<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.2.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/logo.svg" color="#222">

<link rel="stylesheet" href="/css/main.css">

<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&display=swap&subset=latin,latin-ext">
<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"blog.hory-ai.com","root":"/","scheme":"Pisces","version":"7.8.0","exturl":false,"sidebar":{"position":"right","Muse | Mist":320,"display":"always","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":true,"show_result":true,"style":"mac"},"back2top":{"enable":true,"sidebar":false,"scrollpercent":true},"bookmark":{"enable":true,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},"path":"search.xml"};
  </script>

  <meta property="og:type" content="website">
<meta property="og:title" content="Horysk 宏睿时空">
<meta property="og:url" content="http://blog.hory-ai.com/page/2/index.html">
<meta property="og:site_name" content="Horysk 宏睿时空">
<meta property="og:locale" content="en_US">
<meta property="article:author" content="Hory Skone">
<meta name="twitter:card" content="summary">

<link rel="canonical" href="http://blog.hory-ai.com/page/2/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : true,
    isPost : false,
    lang   : 'en'
  };
</script>

  <title>Horysk 宏睿时空</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="Toggle navigation bar">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">Horysk 宏睿时空</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
        <i class="fa fa-search fa-fw fa-lg"></i>
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="main-menu menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-home fa-fw"></i>Home</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-tags fa-fw"></i>Tags</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="fa fa-th fa-fw"></i>Categories</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-archive fa-fw"></i>Archives</a>

  </li>
      <li class="menu-item menu-item-search">
        <a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>Search
        </a>
      </li>
  </ul>
</nav>



  <div class="search-pop-overlay">
    <div class="popup search-popup">
        <div class="search-header">
  <span class="search-icon">
    <i class="fa fa-search"></i>
  </span>
  <div class="search-input-container">
    <input autocomplete="off" autocapitalize="off"
           placeholder="Searching..." spellcheck="false"
           type="search" class="search-input">
  </div>
  <span class="popup-btn-close">
    <i class="fa fa-times-circle"></i>
  </span>
</div>
<div id="search-result">
  <div id="no-result">
    <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
  </div>
</div>

    </div>
  </div>

</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>
  <div class="reading-progress-bar"></div>
  <a role="button" class="book-mark-link book-mark-link-fixed"></a>

  <a href="https://github.com/horysk" class="github-corner" title="Follow me on GitHub" aria-label="Follow me on GitHub" rel="noopener" target="_blank"><svg width="80" height="80" viewBox="0 0 250 250" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content index posts-expand">
            



	<div class="tag-cloud">
	  <div class="tag-cloud-tags" id="tags">
		<a href="/tags/AI/" style="font-size: 16px; color: #fff">AI</a> <a href="/tags/BI/" style="font-size: 16px; color: #fff">BI</a> <a href="/tags/Centos/" style="font-size: 16px; color: #fff">Centos</a> <a href="/tags/DNS/" style="font-size: 16px; color: #fff">DNS</a> <a href="/tags/Dapp/" style="font-size: 16px; color: #fff">Dapp</a> <a href="/tags/Docker/" style="font-size: 16px; color: #fff">Docker</a> <a href="/tags/Hack/" style="font-size: 16px; color: #fff">Hack</a> <a href="/tags/Hexo/" style="font-size: 16px; color: #fff">Hexo</a> <a href="/tags/HyperLedger-Fabric/" style="font-size: 16px; color: #fff">HyperLedger Fabric</a> <a href="/tags/Hyperledger-Fabric/" style="font-size: 16px; color: #fff">Hyperledger Fabric</a> <a href="/tags/Iftop/" style="font-size: 16px; color: #fff">Iftop</a> <a href="/tags/Linux/" style="font-size: 16px; color: #fff">Linux</a> <a href="/tags/ML/" style="font-size: 16px; color: #fff">ML</a> <a href="/tags/MTProxy/" style="font-size: 16px; color: #fff">MTProxy</a> <a href="/tags/Mongo/" style="font-size: 16px; color: #fff">Mongo</a> <a href="/tags/Network/" style="font-size: 16px; color: #fff">Network</a> <a href="/tags/Pyppeteer/" style="font-size: 16px; color: #fff">Pyppeteer</a> <a href="/tags/Tools/" style="font-size: 16px; color: #fff">Tools</a> <a href="/tags/VPN/" style="font-size: 16px; color: #fff">VPN</a> <a href="/tags/Vnc/" style="font-size: 16px; color: #fff">Vnc</a> <a href="/tags/baostock/" style="font-size: 16px; color: #fff">baostock</a> <a href="/tags/block-chain/" style="font-size: 16px; color: #fff">block chain</a> <a href="/tags/centos-xfce-vnc/" style="font-size: 16px; color: #fff">centos-xfce-vnc</a> <a href="/tags/docker/" style="font-size: 16px; color: #fff">docker</a> <a href="/tags/fabric/" style="font-size: 16px; color: #fff">fabric</a> <a href="/tags/hexo/" style="font-size: 16px; color: #fff">hexo</a> <a href="/tags/horysk/" style="font-size: 16px; color: #fff">horysk</a> <a href="/tags/k8s/" style="font-size: 16px; color: #fff">k8s</a> <a href="/tags/linux/" style="font-size: 16px; color: #fff">linux</a> <a href="/tags/mongo/" style="font-size: 16px; color: #fff">mongo</a> <a href="/tags/pyppeteer/" style="font-size: 16px; color: #fff">pyppeteer</a> <a href="/tags/python/" style="font-size: 16px; color: #fff">python</a> <a href="/tags/quant/" style="font-size: 16px; color: #fff">quant</a> <a href="/tags/stock/" style="font-size: 16px; color: #fff">stock</a> <a href="/tags/tushare/" style="font-size: 16px; color: #fff">tushare</a>
	  </div>
	</div>

	
	<script type="text/javascript">
     var alltags = document.getElementsByClassName('tag-cloud-tags');
     var tags = alltags[0].getElementsByTagName('a');
     for (var i = tags.length - 1; i >= 0; i--) {
       var r=Math.floor(Math.random()*75+130);
       var g=Math.floor(Math.random()*75+100);
       var b=Math.floor(Math.random()*75+80);
       tags[i].style.background = "rgb("+r+","+g+","+b+")";
     }
</script>

<style>
  .tag-cloud-tags{
    /*font-family: Helvetica, Tahoma, Arial;*/
    /*font-weight: 100;*/
    text-align: center;
    counter-reset: tags;
  }
  .tag-cloud-tags a{
    border-radius: 6px;
    padding-right: 5px;
    padding-left: 5px;
    margin: 8px 5px 0px 0px;
  }
  .tag-cloud-tags a:before{
    content: "?";
  }

  .tag-cloud-tags a:hover{
     box-shadow: 0px 5px 15px 0px rgba(0,0,0,.4);
     transform: scale(1.1);
     /*box-shadow: 10px 10px 15px 2px rgba(0,0,0,.12), 0 0 6px 0 rgba(104, 104, 105, 0.1);*/
     transition-duration: 0.15s;
  }
</style>

      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/31/Docker%E5%AE%B9%E5%99%A8%E5%91%BD%E4%BB%A4%E6%95%B4%E7%90%86/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          
            <a href="/2020/07/31/Docker%E5%AE%B9%E5%99%A8%E5%91%BD%E4%BB%A4%E6%95%B4%E7%90%86/" class="post-title-link" itemprop="url">Docker容器命令整理</a>
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-31 11:51:20" itemprop="dateCreated datePublished" datetime="2020-07-31T11:51:20+00:00">2020-07-31</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/docker/" itemprop="url" rel="index"><span itemprop="name">docker</span></a>
                </span>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/31/Docker%E5%AE%B9%E5%99%A8%E5%91%BD%E4%BB%A4%E6%95%B4%E7%90%86/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/31/Docker%E5%AE%B9%E5%99%A8%E5%91%BD%E4%BB%A4%E6%95%B4%E7%90%86/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>353</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>1 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
          <p>Docker重启所有命令以及构建是如何设置开机重启</p>
<h2 id="设置开机重启"><a href="#设置开机重启" class="headerlink" title="设置开机重启"></a>设置开机重启</h2><p>–restart=always</p>
<h2 id="启动所有容器"><a href="#启动所有容器" class="headerlink" title="启动所有容器"></a>启动所有容器</h2><p>docker start $(docker ps -a | awk ‘{ print $1}’ | tail -n +2)</p>
          <!--noindex-->
            <div class="post-button">
              <a class="btn" href="/2020/07/31/Docker%E5%AE%B9%E5%99%A8%E5%91%BD%E4%BB%A4%E6%95%B4%E7%90%86/#more" rel="contents">
                Read more &raquo;
              </a>
            </div>
          <!--/noindex-->
        
      
    </div>

    
    
    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  

      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/30/%E5%A4%9A%E6%9C%BA%E7%89%88__docker-compose%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4%E8%AF%A6%E7%BB%86%E6%AD%A5%E9%AA%A4/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          
            <a href="/2020/07/30/%E5%A4%9A%E6%9C%BA%E7%89%88__docker-compose%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4%E8%AF%A6%E7%BB%86%E6%AD%A5%E9%AA%A4/" class="post-title-link" itemprop="url">多机版———docker-compose搭建MongoDB 3.4 分片集群详细步骤</a>
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-30 17:52:25" itemprop="dateCreated datePublished" datetime="2020-07-30T17:52:25+00:00">2020-07-30</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/mongo/" itemprop="url" rel="index"><span itemprop="name">mongo</span></a>
                </span>
                  , 
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/mongo/docker/" itemprop="url" rel="index"><span itemprop="name">docker</span></a>
                </span>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/30/%E5%A4%9A%E6%9C%BA%E7%89%88__docker-compose%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4%E8%AF%A6%E7%BB%86%E6%AD%A5%E9%AA%A4/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/30/%E5%A4%9A%E6%9C%BA%E7%89%88__docker-compose%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4%E8%AF%A6%E7%BB%86%E6%AD%A5%E9%AA%A4/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>12k</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>11 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
          <h2 id="服务器三台-按顺序执行"><a href="#服务器三台-按顺序执行" class="headerlink" title="服务器三台  按顺序执行"></a>服务器三台  按顺序执行</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">192.168</span><span class="number">.1</span><span class="number">.125</span></span><br><span class="line"><span class="number">192.168</span><span class="number">.1</span><span class="number">.126</span></span><br><span class="line"><span class="number">192.168</span><span class="number">.1</span><span class="number">.127</span></span><br></pre></td></tr></table></figure>
<h2 id="创建目录"><a href="#创建目录" class="headerlink" title="创建目录"></a>创建目录</h2><p>在每台机器上操作此步骤</p>
          <!--noindex-->
            <div class="post-button">
              <a class="btn" href="/2020/07/30/%E5%A4%9A%E6%9C%BA%E7%89%88__docker-compose%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4%E8%AF%A6%E7%BB%86%E6%AD%A5%E9%AA%A4/#more" rel="contents">
                Read more &raquo;
              </a>
            </div>
          <!--/noindex-->
        
      
    </div>

    
    
    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  

      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/30/%E5%8D%95%E6%9C%BA%E7%89%88%E2%80%94%E2%80%94docker%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          
            <a href="/2020/07/30/%E5%8D%95%E6%9C%BA%E7%89%88%E2%80%94%E2%80%94docker%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4/" class="post-title-link" itemprop="url">单机版——docker搭建MongoDB分片集群</a>
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-30 12:02:04" itemprop="dateCreated datePublished" datetime="2020-07-30T12:02:04+00:00">2020-07-30</time>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/30/%E5%8D%95%E6%9C%BA%E7%89%88%E2%80%94%E2%80%94docker%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/30/%E5%8D%95%E6%9C%BA%E7%89%88%E2%80%94%E2%80%94docker%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>4.3k</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>4 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
          <p>记录了在如何利用docker搭建MongoDB的分片集群。这里使用1个mongos、3个config server和2组分片副本集，每组分片副本集1主2从。</p>
<h2 id="1-搭建config-server"><a href="#1-搭建config-server" class="headerlink" title="1. 搭建config server"></a>1. 搭建config server</h2><p>这里启动了3个config server。docker命令如下：</p>
          <!--noindex-->
            <div class="post-button">
              <a class="btn" href="/2020/07/30/%E5%8D%95%E6%9C%BA%E7%89%88%E2%80%94%E2%80%94docker%E6%90%AD%E5%BB%BAMongoDB%E5%88%86%E7%89%87%E9%9B%86%E7%BE%A4/#more" rel="contents">
                Read more &raquo;
              </a>
            </div>
          <!--/noindex-->
        
      
    </div>

    
    
    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  

      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/29/%E8%8E%B7%E5%8F%96scrapy-request-%E5%A4%B4%E9%83%A8%E4%BF%A1%E6%81%AF%E5%8F%8A%E6%8D%95%E6%8D%89%E5%BC%82%E5%B8%B8%E4%BF%A1%E6%81%AF/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          
            <a href="/2020/07/29/%E8%8E%B7%E5%8F%96scrapy-request-%E5%A4%B4%E9%83%A8%E4%BF%A1%E6%81%AF%E5%8F%8A%E6%8D%95%E6%8D%89%E5%BC%82%E5%B8%B8%E4%BF%A1%E6%81%AF/" class="post-title-link" itemprop="url">获取scrapy request 头部信息及捕捉异常信息</a>
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-29 17:35:56" itemprop="dateCreated datePublished" datetime="2020-07-29T17:35:56+00:00">2020-07-29</time>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/29/%E8%8E%B7%E5%8F%96scrapy-request-%E5%A4%B4%E9%83%A8%E4%BF%A1%E6%81%AF%E5%8F%8A%E6%8D%95%E6%8D%89%E5%BC%82%E5%B8%B8%E4%BF%A1%E6%81%AF/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/29/%E8%8E%B7%E5%8F%96scrapy-request-%E5%A4%B4%E9%83%A8%E4%BF%A1%E6%81%AF%E5%8F%8A%E6%8D%95%E6%8D%89%E5%BC%82%E5%B8%B8%E4%BF%A1%E6%81%AF/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>22k</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>20 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
          <p>获取scrapy request 头部信息 及捕捉异常信息</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"><span class="keyword">import</span> scrapy</span><br><span class="line"><span class="keyword">from</span> Amazon.items <span class="keyword">import</span> AmazonItem</span><br><span class="line"><span class="keyword">from</span> scrapy_redis_sentinel.spiders <span class="keyword">import</span> RedisSpider</span><br><span class="line"><span class="keyword">from</span> Amazon.tools <span class="keyword">import</span> GetAmazonUrlCountry , GetLanguage,replace_string, replace_list, formtting_rank,formtting_info,GetAmazonUrlDomain,GetAmazonUrlCID</span><br><span class="line"><span class="keyword">from</span> datetime <span class="keyword">import</span> datetime</span><br><span class="line"><span class="keyword">import</span> urllib3</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> dateutil.parser <span class="keyword">import</span> *</span><br><span class="line"><span class="keyword">from</span> Amazon.ConfigDB <span class="keyword">import</span> RedisDB,MongoDB</span><br><span class="line">mongo_conn=MongoDB(<span class="string">&quot;AmazonDB&quot;</span>,<span class="string">&quot;AmazonAddKW&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">AmazonkwSpider</span>(<span class="params">RedisSpider</span>):</span></span><br><span class="line">    name = <span class="string">&#x27;amazonkw&#x27;</span></span><br><span class="line">    redis_key = <span class="string">&quot;amazonkw&quot;</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># def start_requests(self):</span></span><br><span class="line">    <span class="comment">#     url=&quot;http://httpbin.org/ip&quot;</span></span><br><span class="line">    <span class="comment">#     yield scrapy.Request(url,  callback=self.parse)</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse</span>(<span class="params">self, response</span>):</span></span><br><span class="line">        <span class="comment"># res=response.request.headers</span></span><br><span class="line">        <span class="comment"># print(res)</span></span><br><span class="line">        <span class="comment"># print(response.text)</span></span><br></pre></td></tr></table></figure>
<h2 id="scrapy-middleware中间件"><a href="#scrapy-middleware中间件" class="headerlink" title="scrapy  middleware中间件"></a>scrapy  middleware中间件</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br><span class="line">203</span><br><span class="line">204</span><br><span class="line">205</span><br><span class="line">206</span><br><span class="line">207</span><br><span class="line">208</span><br><span class="line">209</span><br><span class="line">210</span><br><span class="line">211</span><br><span class="line">212</span><br><span class="line">213</span><br><span class="line">214</span><br><span class="line">215</span><br><span class="line">216</span><br><span class="line">217</span><br><span class="line">218</span><br><span class="line">219</span><br><span class="line">220</span><br><span class="line">221</span><br><span class="line">222</span><br><span class="line">223</span><br><span class="line">224</span><br><span class="line">225</span><br><span class="line">226</span><br><span class="line">227</span><br><span class="line">228</span><br><span class="line">229</span><br><span class="line">230</span><br><span class="line">231</span><br><span class="line">232</span><br><span class="line">233</span><br><span class="line">234</span><br><span class="line">235</span><br><span class="line">236</span><br><span class="line">237</span><br><span class="line">238</span><br><span class="line">239</span><br><span class="line">240</span><br><span class="line">241</span><br><span class="line">242</span><br><span class="line">243</span><br><span class="line">244</span><br><span class="line">245</span><br><span class="line">246</span><br><span class="line">247</span><br><span class="line">248</span><br><span class="line">249</span><br><span class="line">250</span><br><span class="line">251</span><br><span class="line">252</span><br><span class="line">253</span><br><span class="line">254</span><br><span class="line">255</span><br><span class="line">256</span><br><span class="line">257</span><br><span class="line">258</span><br><span class="line">259</span><br><span class="line">260</span><br><span class="line">261</span><br><span class="line">262</span><br><span class="line">263</span><br><span class="line">264</span><br><span class="line">265</span><br><span class="line">266</span><br><span class="line">267</span><br><span class="line">268</span><br><span class="line">269</span><br><span class="line">270</span><br><span class="line">271</span><br><span class="line">272</span><br><span class="line">273</span><br><span class="line">274</span><br><span class="line">275</span><br><span class="line">276</span><br><span class="line">277</span><br><span class="line">278</span><br><span class="line">279</span><br><span class="line">280</span><br><span class="line">281</span><br><span class="line">282</span><br><span class="line">283</span><br><span class="line">284</span><br><span class="line">285</span><br><span class="line">286</span><br><span class="line">287</span><br><span class="line">288</span><br><span class="line">289</span><br><span class="line">290</span><br><span class="line">291</span><br><span class="line">292</span><br><span class="line">293</span><br><span class="line">294</span><br><span class="line">295</span><br><span class="line">296</span><br><span class="line">297</span><br><span class="line">298</span><br><span class="line">299</span><br><span class="line">300</span><br><span class="line">301</span><br><span class="line">302</span><br><span class="line">303</span><br><span class="line">304</span><br><span class="line">305</span><br><span class="line">306</span><br><span class="line">307</span><br><span class="line">308</span><br><span class="line">309</span><br><span class="line">310</span><br><span class="line">311</span><br><span class="line">312</span><br><span class="line">313</span><br><span class="line">314</span><br><span class="line">315</span><br><span class="line">316</span><br><span class="line">317</span><br><span class="line">318</span><br><span class="line">319</span><br><span class="line">320</span><br><span class="line">321</span><br><span class="line">322</span><br><span class="line">323</span><br><span class="line">324</span><br><span class="line">325</span><br><span class="line">326</span><br><span class="line">327</span><br><span class="line">328</span><br><span class="line">329</span><br><span class="line">330</span><br><span class="line">331</span><br><span class="line">332</span><br><span class="line">333</span><br><span class="line">334</span><br><span class="line">335</span><br><span class="line">336</span><br><span class="line">337</span><br><span class="line">338</span><br><span class="line">339</span><br><span class="line">340</span><br><span class="line">341</span><br><span class="line">342</span><br><span class="line">343</span><br><span class="line">344</span><br><span class="line">345</span><br><span class="line">346</span><br><span class="line">347</span><br><span class="line">348</span><br><span class="line">349</span><br><span class="line">350</span><br><span class="line">351</span><br><span class="line">352</span><br><span class="line">353</span><br><span class="line">354</span><br><span class="line">355</span><br><span class="line">356</span><br><span class="line">357</span><br><span class="line">358</span><br><span class="line">359</span><br><span class="line">360</span><br><span class="line">361</span><br><span class="line">362</span><br><span class="line">363</span><br><span class="line">364</span><br><span class="line">365</span><br><span class="line">366</span><br><span class="line">367</span><br><span class="line">368</span><br><span class="line">369</span><br><span class="line">370</span><br><span class="line">371</span><br><span class="line">372</span><br><span class="line">373</span><br><span class="line">374</span><br><span class="line">375</span><br><span class="line">376</span><br><span class="line">377</span><br><span class="line">378</span><br><span class="line">379</span><br><span class="line">380</span><br><span class="line">381</span><br><span class="line">382</span><br><span class="line">383</span><br><span class="line">384</span><br><span class="line">385</span><br><span class="line">386</span><br><span class="line">387</span><br><span class="line">388</span><br><span class="line">389</span><br><span class="line">390</span><br><span class="line">391</span><br><span class="line">392</span><br><span class="line">393</span><br><span class="line">394</span><br><span class="line">395</span><br><span class="line">396</span><br><span class="line">397</span><br><span class="line">398</span><br><span class="line">399</span><br><span class="line">400</span><br><span class="line">401</span><br><span class="line">402</span><br><span class="line">403</span><br><span class="line">404</span><br><span class="line">405</span><br><span class="line">406</span><br><span class="line">407</span><br><span class="line">408</span><br><span class="line">409</span><br><span class="line">410</span><br><span class="line">411</span><br><span class="line">412</span><br><span class="line">413</span><br><span class="line">414</span><br><span class="line">415</span><br><span class="line">416</span><br><span class="line">417</span><br><span class="line">418</span><br><span class="line">419</span><br><span class="line">420</span><br><span class="line">421</span><br><span class="line">422</span><br><span class="line">423</span><br><span class="line">424</span><br><span class="line">425</span><br><span class="line">426</span><br><span class="line">427</span><br><span class="line">428</span><br><span class="line">429</span><br><span class="line">430</span><br><span class="line">431</span><br><span class="line">432</span><br><span class="line">433</span><br><span class="line">434</span><br><span class="line">435</span><br><span class="line">436</span><br><span class="line">437</span><br><span class="line">438</span><br><span class="line">439</span><br><span class="line">440</span><br><span class="line">441</span><br><span class="line">442</span><br><span class="line">443</span><br><span class="line">444</span><br><span class="line">445</span><br><span class="line">446</span><br><span class="line">447</span><br><span class="line">448</span><br><span class="line">449</span><br><span class="line">450</span><br><span class="line">451</span><br><span class="line">452</span><br><span class="line">453</span><br><span class="line">454</span><br><span class="line">455</span><br><span class="line">456</span><br><span class="line">457</span><br><span class="line">458</span><br><span class="line">459</span><br><span class="line">460</span><br><span class="line">461</span><br><span class="line">462</span><br><span class="line">463</span><br><span class="line">464</span><br><span class="line">465</span><br><span class="line">466</span><br><span class="line">467</span><br><span class="line">468</span><br><span class="line">469</span><br><span class="line">470</span><br><span class="line">471</span><br><span class="line">472</span><br><span class="line">473</span><br><span class="line">474</span><br><span class="line">475</span><br><span class="line">476</span><br><span class="line">477</span><br><span class="line">478</span><br><span class="line">479</span><br><span class="line">480</span><br><span class="line">481</span><br><span class="line">482</span><br><span class="line">483</span><br><span class="line">484</span><br><span class="line">485</span><br><span class="line">486</span><br><span class="line">487</span><br><span class="line">488</span><br><span class="line">489</span><br><span class="line">490</span><br><span class="line">491</span><br><span class="line">492</span><br><span class="line">493</span><br><span class="line">494</span><br><span class="line">495</span><br><span class="line">496</span><br><span class="line">497</span><br><span class="line">498</span><br><span class="line">499</span><br><span class="line">500</span><br><span class="line">501</span><br><span class="line">502</span><br><span class="line">503</span><br><span class="line">504</span><br><span class="line">505</span><br><span class="line">506</span><br><span class="line">507</span><br><span class="line">508</span><br><span class="line">509</span><br><span class="line">510</span><br><span class="line">511</span><br><span class="line">512</span><br><span class="line">513</span><br><span class="line">514</span><br><span class="line">515</span><br><span class="line">516</span><br><span class="line">517</span><br><span class="line">518</span><br><span class="line">519</span><br><span class="line">520</span><br><span class="line">521</span><br><span class="line">522</span><br><span class="line">523</span><br><span class="line">524</span><br><span class="line">525</span><br><span class="line">526</span><br><span class="line">527</span><br><span class="line">528</span><br><span class="line">529</span><br><span class="line">530</span><br><span class="line">531</span><br><span class="line">532</span><br><span class="line">533</span><br><span class="line">534</span><br><span class="line">535</span><br><span class="line">536</span><br><span class="line">537</span><br><span class="line">538</span><br><span class="line">539</span><br><span class="line">540</span><br><span class="line">541</span><br><span class="line">542</span><br><span class="line">543</span><br><span class="line">544</span><br><span class="line">545</span><br><span class="line">546</span><br><span class="line">547</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># -*- coding: utf-8 -*-</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># Define here the models for your spider middleware</span></span><br><span class="line"><span class="comment">#</span></span><br><span class="line"><span class="comment"># See documentation in:</span></span><br><span class="line"><span class="comment"># https://docs.scrapy.org/en/latest/topics/spider-middleware.html</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> signals</span><br><span class="line"><span class="keyword">from</span> datetime <span class="keyword">import</span> datetime</span><br><span class="line"><span class="keyword">from</span> dateutil.parser <span class="keyword">import</span> *</span><br><span class="line"><span class="keyword">from</span> Amazon.ConfigDB <span class="keyword">import</span> RedisDB,RedisPool,MongoDB</span><br><span class="line"><span class="keyword">from</span> Amazon.tools <span class="keyword">import</span> GetAmazonUrlCountry , GetLanguage,replace_string, replace_list, formtting_rank,formtting_info,GetAmazonUrlDomain,GetAmazonUrlCID</span><br><span class="line">redis_ua=RedisPool()</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">AmazonSpiderMiddleware</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line">    <span class="comment"># Not all methods need to be defined. If a method is not defined,</span></span><br><span class="line">    <span class="comment"># scrapy acts as if the spider middleware does not modify the</span></span><br><span class="line">    <span class="comment"># passed objects.</span></span><br><span class="line"></span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span>(<span class="params">cls, crawler</span>):</span></span><br><span class="line">        <span class="comment"># This method is used by Scrapy to create your spiders.</span></span><br><span class="line">        s = cls()</span><br><span class="line">        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)</span><br><span class="line">        <span class="keyword">return</span> s</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_spider_input</span>(<span class="params">self, response, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called for each response that goes through the spider</span></span><br><span class="line">        <span class="comment"># middleware and into the spider.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Should return None or raise an exception.</span></span><br><span class="line">        <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_spider_output</span>(<span class="params">self, response, result, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called with the results returned from the Spider, after</span></span><br><span class="line">        <span class="comment"># it has processed the response.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Must return an iterable of Request, dict or Item objects.</span></span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> result:</span><br><span class="line">            <span class="keyword">yield</span> i</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_spider_exception</span>(<span class="params">self, response, exception, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called when a spider or process_spider_input() method</span></span><br><span class="line">        <span class="comment"># (from other spider middleware) raises an exception.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Should return either None or an iterable of Request, dict</span></span><br><span class="line">        <span class="comment"># or Item objects.</span></span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_start_requests</span>(<span class="params">self, start_requests, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called with the start requests of the spider, and works</span></span><br><span class="line">        <span class="comment"># similarly to the process_spider_output() method, except</span></span><br><span class="line">        <span class="comment"># that it doesn’t have a response associated.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Must return only requests (not items).</span></span><br><span class="line">        <span class="keyword">for</span> r <span class="keyword">in</span> start_requests:</span><br><span class="line">            <span class="keyword">yield</span> r</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">spider_opened</span>(<span class="params">self, spider</span>):</span></span><br><span class="line">        spider.logger.info(<span class="string">&#x27;Spider opened: %s&#x27;</span> % spider.name)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">DownloaderMiddleware</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line">    <span class="comment"># Not all methods need to be defined. If a method is not defined,</span></span><br><span class="line">    <span class="comment"># scrapy acts as if the downloader middleware does not modify the</span></span><br><span class="line">    <span class="comment"># passed objects.</span></span><br><span class="line"></span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span>(<span class="params">cls, crawler</span>):</span></span><br><span class="line">        <span class="comment"># This method is used by Scrapy to create your spiders.</span></span><br><span class="line">        s = cls()</span><br><span class="line">        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)</span><br><span class="line">        <span class="keyword">return</span> s</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_request</span>(<span class="params">self, request, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called for each request that goes through the downloader</span></span><br><span class="line">        <span class="comment"># middleware.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Must either:</span></span><br><span class="line">        <span class="comment"># - return None: continue processing this request</span></span><br><span class="line">        <span class="comment"># - or return a Response object</span></span><br><span class="line">        <span class="comment"># - or return a Request object</span></span><br><span class="line">        <span class="comment"># - or raise IgnoreRequest: process_exception() methods of</span></span><br><span class="line">        <span class="comment">#   installed downloader middleware will be called</span></span><br><span class="line">        <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_response</span>(<span class="params">self, request, response, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called with the response returned from the downloader.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Must either;</span></span><br><span class="line">        <span class="comment"># - return a Response object</span></span><br><span class="line">        <span class="comment"># - return a Request object</span></span><br><span class="line">        <span class="comment"># - or raise IgnoreRequest</span></span><br><span class="line">        <span class="keyword">return</span> response</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_exception</span>(<span class="params">self, request, exception, spider</span>):</span></span><br><span class="line">        <span class="comment"># Called when a download handler or a process_request()</span></span><br><span class="line">        <span class="comment"># (from other downloader middleware) raises an exception.</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Must either:</span></span><br><span class="line">        <span class="comment"># - return None: continue processing this exception</span></span><br><span class="line">        <span class="comment"># - return a Response object: stops process_exception() chain</span></span><br><span class="line">        <span class="comment"># - return a Request object: stops process_exception() chain</span></span><br><span class="line">        <span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">spider_opened</span>(<span class="params">self, spider</span>):</span></span><br><span class="line">        spider.logger.info(<span class="string">&#x27;Spider opened: %s&#x27;</span> % spider.name)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> base64</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="comment"># from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware</span></span><br><span class="line"><span class="keyword">from</span> Amazon.settings <span class="keyword">import</span> USER_AGENTS</span><br><span class="line"><span class="comment"># from settings import PROXIES</span></span><br><span class="line"><span class="comment"># class RandomUserAgent(UserAgentMiddleware):</span></span><br><span class="line"><span class="comment">#     def process_request(self, request, spider):</span></span><br><span class="line"><span class="comment">#         _ua = random.choice(USER_AGENTS)</span></span><br><span class="line"><span class="comment">#         request.headers.setdefault(&#x27;User-Agent&#x27;, _ua)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># class ProxyMiddleware(object):</span></span><br><span class="line"><span class="comment"># 	def process_request(self, request, spider):</span></span><br><span class="line"><span class="comment">#         proxy = requests.get(&quot;http://172.20.0.252:5010/get/&quot;)</span></span><br><span class="line">		<span class="comment"># proxy = random.choice(PROXIES)print(proxy,&quot;*&quot;*100)</span></span><br><span class="line">		<span class="comment"># if proxy[&#x27;user_pass&#x27;] is not None:</span></span><br><span class="line">		<span class="comment"># 	request.meta[&#x27;proxy&#x27;] = &quot;http://%s&quot; % proxy[&#x27;ip_port&#x27;]</span></span><br><span class="line">		<span class="comment"># 	encoded_user_pass = base64.encodestring(proxy[&#x27;user_pass&#x27;])</span></span><br><span class="line">		<span class="comment"># 	request.headers[&#x27;Proxy-Authorization&#x27;] = &#x27;Basic &#x27; + encoded_user_pass</span></span><br><span class="line">		<span class="comment"># 	print (&quot;**************ProxyMiddleware have pass************&quot;) + proxy[&#x27;ip_port&#x27;]</span></span><br><span class="line">		<span class="comment"># else:</span></span><br><span class="line">		<span class="comment"># 	# print (&quot;**************ProxyMiddleware no pass************&quot;) + proxy[&#x27;ip_port&#x27;]</span></span><br><span class="line">		<span class="comment"># 	request.meta[&#x27;proxy&#x27;] = &quot;http://%s&quot; % proxy[&#x27;ip_port&#x27;]</span></span><br><span class="line">        </span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> signals</span><br><span class="line"><span class="keyword">from</span> w3lib.http <span class="keyword">import</span> basic_auth_header</span><br><span class="line"><span class="keyword">import</span> base64</span><br><span class="line"><span class="keyword">from</span> scrapy.downloadermiddlewares.retry <span class="keyword">import</span> RetryMiddleware</span><br><span class="line"></span><br><span class="line"><span class="comment"># 代理服务器</span></span><br><span class="line">proxyServer = <span class="string">&quot;http://http-dyn.abuyun.com:9020&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 代理隧道验证信息</span></span><br><span class="line">proxyUser = <span class="string">&quot;H6587BH09900664D&quot;</span></span><br><span class="line">proxyPass = <span class="string">&quot;20C4314AF6C62E0F&quot;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># for Python2</span></span><br><span class="line"><span class="comment"># proxyAuth = &quot;Basic &quot; + base64.b64encode(proxyUser + &quot;:&quot; + proxyPass)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># for Python3</span></span><br><span class="line">proxyAuth = <span class="string">&quot;Basic &quot;</span> + base64.urlsafe_b64encode(<span class="built_in">bytes</span>((proxyUser + <span class="string">&quot;:&quot;</span> + proxyPass), <span class="string">&quot;ascii&quot;</span>)).decode(<span class="string">&quot;utf8&quot;</span>)</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">RandomProxy</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_request</span>(<span class="params">self, request, spider</span>):</span></span><br><span class="line">        request.meta[<span class="string">&quot;proxy&quot;</span>] = proxyServer</span><br><span class="line">        self.ua =  redis_ua.sget(<span class="string">&quot;user-agent&quot;</span>)</span><br><span class="line">        request.headers[<span class="string">&quot;User-Agent&quot;</span>]=<span class="built_in">str</span>(self.ua,encoding=<span class="string">&quot;utf-8&quot;</span>)</span><br><span class="line">        request.headers[<span class="string">&quot;Proxy-Authorization&quot;</span>] = proxyAuth  </span><br><span class="line">        <span class="comment"># print(request)</span></span><br><span class="line">        request.cookies=&#123;</span><br><span class="line">            <span class="string">&quot;session-id&quot;</span>:<span class="string">&quot;134-0075797-0381056&quot;</span>,</span><br><span class="line">            <span class="string">&quot;session-id-time&quot;</span>:<span class="string">&quot;2082787201l&quot;</span>,</span><br><span class="line">            <span class="string">&quot;i18n-prefs&quot;</span>:<span class="string">&quot;USD&quot;</span>,</span><br><span class="line">            <span class="string">&quot;lc-main&quot;</span>:<span class="string">&quot;en_US&quot;</span>,</span><br><span class="line">            <span class="string">&quot;sp-cdn&quot;</span>:<span class="string">&quot;L5Z9:CN&quot;</span>,</span><br><span class="line">            <span class="string">&quot;ubid-main&quot;</span>:<span class="string">&quot;134-0448665-3732200&quot;</span>,</span><br><span class="line">            <span class="string">&quot;session-token&quot;</span>:<span class="string">&quot;KBeUZ2ocx7ObasfCr4XpPQE7t5RPj80VGzKb+EbFEEAMWbzwl8YuNtcPmqhZ6kUUtVxpwwOJSOy0XvZvPIKuxoUIgKbIuo6Z3qaX1Lk/0ewvaJY55aR+ayz/kJ9jUmL2HhLMwE8ipxIPqc/lzRRpAUxD4DuCd2PidfXY+sG5IhTVooAuK4UMzq1gB3Azo/BO&quot;</span></span><br><span class="line">        &#125;  </span><br><span class="line">    <span class="comment"># def process_request(self, request, spider):</span></span><br><span class="line">        <span class="comment"># proxy = &quot;tps121.kdlapi.com:15818&quot;</span></span><br><span class="line">        <span class="comment"># request.meta[&#x27;proxy&#x27;] = &quot;https://%s&quot; %proxy</span></span><br><span class="line">        <span class="comment"># # 用户名密码认证</span></span><br><span class="line">        <span class="comment"># request.headers[&#x27;Proxy-Authorization&#x27;] = basic_auth_header(&#x27;t19351016044852&#x27;, &#x27;y4asuamy&#x27;)  # 白名单认证可注释此行</span></span><br><span class="line">        <span class="comment"># return None</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># def __init__(self):</span></span><br><span class="line">    <span class="comment">#     self.proxy =  redis_ua.sget(&quot;daxiangIP&quot;)</span></span><br><span class="line">        </span><br><span class="line">    <span class="comment"># def process_request(self,request, spider):</span></span><br><span class="line">        <span class="comment"># proxy = requests.get(&quot;http://172.20.0.252:5010/get/&quot;).content</span></span><br><span class="line">        <span class="comment"># proxy=eval(str(proxy,encoding=&quot;utf-8&quot;))[&quot;proxy&quot;]</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># print(&quot;ip_proxy=&#123;&#125;&quot;.format(str(self.proxy,encoding=&quot;utf8&quot;)),&quot;*&quot;*30)</span></span><br><span class="line">        <span class="comment"># request.meta[&#x27;proxy&#x27;] = &#x27;https://27.188.65.244:8060&#x27;</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># from fake_useragent import UserAgent</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">RandomUserAgentMiddlware</span>(<span class="params"><span class="built_in">object</span></span>):</span></span><br><span class="line">    <span class="comment">#随机更换user-agent</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span>(<span class="params">self,crawler</span>):</span></span><br><span class="line">        <span class="built_in">super</span>(RandomUserAgentMiddlware,self).__init__()</span><br><span class="line">        <span class="comment"># self.ua =  random.choice(USER_AGENTS)</span></span><br><span class="line">        self.ua =  redis_ua.sget(<span class="string">&quot;user-agent&quot;</span>)</span><br><span class="line">        <span class="comment"># redis_ua.rpush(&#x27;user-agent&#x27;,self.ua)</span></span><br><span class="line">        self.ua=<span class="built_in">str</span>(self.ua,encoding=<span class="string">&#x27;utf-8&#x27;</span>)</span><br><span class="line">        <span class="comment"># self.ua =  UserAgent(verify_ssl=False)</span></span><br><span class="line"></span><br><span class="line"><span class="meta">    @classmethod</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span>(<span class="params">cls,crawler</span>):</span></span><br><span class="line">        <span class="keyword">return</span> cls(crawler)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_request</span>(<span class="params">self,request,spider</span>):</span></span><br><span class="line">        request.headers.setdefault(<span class="string">&quot;User-Agent&quot;</span>,self.ua)</span><br><span class="line">        <span class="comment"># request.headers.setdefault(&quot;User-Agent&quot;,self.ua.random)</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">GetFailedUrl</span>(<span class="params">RetryMiddleware</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span>(<span class="params">self, settings</span>):</span></span><br><span class="line">        self.max_retry_times = settings.getint(<span class="string">&#x27;RETRY_TIMES&#x27;</span>)</span><br><span class="line">        self.retry_http_codes = <span class="built_in">set</span>(<span class="built_in">int</span>(x) <span class="keyword">for</span> x <span class="keyword">in</span> settings.getlist(<span class="string">&#x27;RETRY_HTTP_CODES&#x27;</span>))</span><br><span class="line">        self.priority_adjust = settings.getint(<span class="string">&#x27;RETRY_PRIORITY_ADJUST&#x27;</span>)</span><br><span class="line"> </span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_response</span>(<span class="params">self, request, response, spider</span>):</span></span><br><span class="line">        <span class="comment"># print(response.text)</span></span><br><span class="line">        <span class="keyword">if</span> response.status != <span class="number">200</span> <span class="keyword">or</span> <span class="string">R&quot;robot check&quot;</span> <span class="keyword">in</span> response.text:</span><br><span class="line">            <span class="keyword">with</span> <span class="built_in">open</span>(<span class="built_in">str</span>(spider.name) + <span class="string">&quot;.txt&quot;</span>, <span class="string">&quot;a&quot;</span>) <span class="keyword">as</span> f:</span><br><span class="line">                f.write(response.url + <span class="string">&quot;\n&quot;</span>)</span><br><span class="line">            <span class="comment"># print(response.url,&quot;*&quot;*100)</span></span><br><span class="line">            <span class="keyword">if</span> <span class="string">R&quot;www.amazon.com/s?k=&quot;</span> <span class="keyword">in</span> request_url:</span><br><span class="line">                db = MongoDB(<span class="string">&quot;AddTask&quot;</span>,<span class="string">&quot;Add_List&quot;</span>)</span><br><span class="line">                items=&#123;&#125;</span><br><span class="line">                items[<span class="string">&quot;cid&quot;</span>]=GetAmazonUrlCID(request_url)</span><br><span class="line">                items[<span class="string">&#x27;kwURL&#x27;</span>]=[]</span><br><span class="line">                items[<span class="string">&#x27;Finished_time&#x27;</span>]=parse(<span class="built_in">str</span>(datetime.now())[<span class="number">0</span>:<span class="number">19</span>])</span><br><span class="line">                items[<span class="string">&#x27;link_url&#x27;</span>]=request_url</span><br><span class="line">                items[<span class="string">&#x27;status&#x27;</span>]=<span class="string">&quot;failed&quot;</span></span><br><span class="line">                items[<span class="string">&#x27;error&#x27;</span>]=<span class="built_in">str</span>(exception)</span><br><span class="line">                <span class="keyword">if</span> db.getInfo(&#123;<span class="string">&#x27;id&#x27;</span>:<span class="built_in">int</span>(items[<span class="string">&quot;cid&quot;</span>])&#125;):</span><br><span class="line">                    db.updateDict(&#123;<span class="string">&#x27;id&#x27;</span>:<span class="built_in">int</span>(items[<span class="string">&#x27;cid&#x27;</span>])&#125;,&#123;<span class="string">&#x27;$set&#x27;</span>:items&#125;)</span><br><span class="line">                <span class="comment"># print(&quot;*&quot;*100,str(exception))</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                db_ = MongoDB(<span class="string">&quot;AmazonDB&quot;</span>,<span class="string">&quot;AmazonContent&quot;</span>)</span><br><span class="line">                items=&#123;&#125;</span><br><span class="line">                items[<span class="string">&#x27;id&#x27;</span>]=db_.getID(<span class="string">&quot;amazonUrl&quot;</span>)</span><br><span class="line">                items[<span class="string">&quot;cid&quot;</span>]=GetAmazonUrlCID(request_url)</span><br><span class="line">                items[<span class="string">&#x27;Finished_time&#x27;</span>]=parse(<span class="built_in">str</span>(datetime.now())[<span class="number">0</span>:<span class="number">19</span>])</span><br><span class="line">                items[<span class="string">&#x27;link_url&#x27;</span>]=request_url</span><br><span class="line">                items[<span class="string">&#x27;status&#x27;</span>]=<span class="string">&quot;failed&quot;</span></span><br><span class="line">                items[<span class="string">&#x27;error&#x27;</span>]=<span class="built_in">str</span>(exception)</span><br><span class="line">                db_.insertDict(items)</span><br><span class="line">            <span class="keyword">return</span> request</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="keyword">return</span> response</span><br><span class="line"> </span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_exception</span>(<span class="params">self, request, exception, spider</span>):</span></span><br><span class="line">        <span class="keyword">if</span> <span class="built_in">isinstance</span>(exception, self.EXCEPTIONS_TO_RETRY):</span><br><span class="line">            request_url=<span class="built_in">str</span>(request)[<span class="number">5</span>:-<span class="number">1</span>]</span><br><span class="line">            <span class="keyword">with</span> <span class="built_in">open</span>(<span class="built_in">str</span>(spider.name) + <span class="string">&quot;.txt&quot;</span>, <span class="string">&quot;a&quot;</span>) <span class="keyword">as</span> f:</span><br><span class="line">                f.write(request_url + <span class="string">&quot;\n&quot;</span>)</span><br><span class="line">                print(<span class="string">&quot;*&quot;</span>*<span class="number">20</span>,request_url)</span><br><span class="line">            <span class="keyword">if</span> <span class="string">R&quot;www.amazon.com/s?k=&quot;</span> <span class="keyword">in</span> request_url:</span><br><span class="line">                db = MongoDB(<span class="string">&quot;AddTask&quot;</span>,<span class="string">&quot;Add_List&quot;</span>)</span><br><span class="line">                items=&#123;&#125;</span><br><span class="line">                items[<span class="string">&quot;cid&quot;</span>]=GetAmazonUrlCID(request_url)</span><br><span class="line">                items[<span class="string">&#x27;kwURL&#x27;</span>]=[]</span><br><span class="line">                items[<span class="string">&#x27;Finished_time&#x27;</span>]=parse(<span class="built_in">str</span>(datetime.now())[<span class="number">0</span>:<span class="number">19</span>])</span><br><span class="line">                items[<span class="string">&#x27;link_url&#x27;</span>]=request_url</span><br><span class="line">                items[<span class="string">&#x27;status&#x27;</span>]=<span class="string">&quot;failed&quot;</span></span><br><span class="line">                items[<span class="string">&#x27;error&#x27;</span>]=<span class="built_in">str</span>(exception)</span><br><span class="line">                <span class="keyword">if</span> db.getInfo(&#123;<span class="string">&#x27;id&#x27;</span>:<span class="built_in">int</span>(items[<span class="string">&quot;cid&quot;</span>])&#125;):</span><br><span class="line">                    db.updateDict(&#123;<span class="string">&#x27;id&#x27;</span>:<span class="built_in">int</span>(items[<span class="string">&#x27;cid&#x27;</span>])&#125;,&#123;<span class="string">&#x27;$set&#x27;</span>:items&#125;)</span><br><span class="line">                <span class="comment"># print(&quot;*&quot;*100,str(exception))</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                db_ = MongoDB(<span class="string">&quot;AmazonDB&quot;</span>,<span class="string">&quot;AmazonContent&quot;</span>)</span><br><span class="line">                items=&#123;&#125;</span><br><span class="line">                items[<span class="string">&#x27;id&#x27;</span>]=db_.getID(<span class="string">&quot;amazonUrl&quot;</span>)</span><br><span class="line">                items[<span class="string">&quot;cid&quot;</span>]=GetAmazonUrlCID(request_url)</span><br><span class="line">                items[<span class="string">&#x27;Finished_time&#x27;</span>]=parse(<span class="built_in">str</span>(datetime.now())[<span class="number">0</span>:<span class="number">19</span>])</span><br><span class="line">                items[<span class="string">&#x27;link_url&#x27;</span>]=request_url</span><br><span class="line">                items[<span class="string">&#x27;status&#x27;</span>]=<span class="string">&quot;failed&quot;</span></span><br><span class="line">                items[<span class="string">&#x27;error&#x27;</span>]=<span class="built_in">str</span>(exception)</span><br><span class="line">                db_.insertDict(items)</span><br><span class="line">            <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># from scrapy import signals</span></span><br><span class="line"><span class="comment"># import pyppeteer</span></span><br><span class="line"><span class="comment"># import asyncio</span></span><br><span class="line"><span class="comment"># import os</span></span><br><span class="line"><span class="comment"># import json</span></span><br><span class="line"><span class="comment"># import redis</span></span><br><span class="line"><span class="comment"># from scrapy.http import HtmlResponse</span></span><br><span class="line"><span class="comment"># from Amazon.ConfigDB import RedisDB,RedisPool,MongoDB</span></span><br><span class="line"><span class="comment"># import logging</span></span><br><span class="line"><span class="comment"># pyppeteer_level = logging.WARNING</span></span><br><span class="line"><span class="comment"># logging.getLogger(&#x27;pyppeteer&#x27;).setLevel(pyppeteer_level)</span></span><br><span class="line"><span class="comment"># logging.getLogger(&#x27;websockets.protocol&#x27;).setLevel(pyppeteer_level)</span></span><br><span class="line"><span class="comment"># pyppeteer_logger = logging.getLogger(&#x27;pyppeteer&#x27;)</span></span><br><span class="line"><span class="comment"># pyppeteer_logger.setLevel(logging.WARNING)</span></span><br><span class="line"><span class="comment"># # redisconn=RedisDB(db=0)</span></span><br><span class="line"><span class="comment"># redis_ua = RedisPool()</span></span><br><span class="line"><span class="comment"># pyppeteer.DEBUG = False</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># def _patch_pyppeteer():</span></span><br><span class="line"><span class="comment">#     from typing import Any</span></span><br><span class="line"><span class="comment">#     from pyppeteer import connection, launcher</span></span><br><span class="line"><span class="comment">#     import websockets.client</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     class PatchedConnection(connection.Connection):  # type: ignore</span></span><br><span class="line"><span class="comment">#         def __init__(self, *args: Any, **kwargs: Any) -&gt; None:</span></span><br><span class="line"><span class="comment">#             super().__init__(*args, **kwargs)</span></span><br><span class="line"><span class="comment">#             # the _ws argument is not yet connected, can simply be replaced with another</span></span><br><span class="line"><span class="comment">#             # with better defaults.</span></span><br><span class="line"><span class="comment">#             self._ws = websockets.client.connect(</span></span><br><span class="line"><span class="comment">#                 self._url,</span></span><br><span class="line"><span class="comment">#                 loop=self._loop,</span></span><br><span class="line"><span class="comment">#                 # the following parameters are all passed to WebSocketCommonProtocol</span></span><br><span class="line"><span class="comment">#                 # which markes all three as Optional, but connect() doesn&#x27;t, hence the liberal</span></span><br><span class="line"><span class="comment">#                 # use of type: ignore on these lines.</span></span><br><span class="line"><span class="comment">#                 # fixed upstream but not yet released, see aaugustin/websockets#93ad88</span></span><br><span class="line"><span class="comment">#                 max_size=None,  # type: ignore</span></span><br><span class="line"><span class="comment">#                 ping_interval=None,  # type: ignore</span></span><br><span class="line"><span class="comment">#                 ping_timeout=None,  # type: ignore</span></span><br><span class="line"><span class="comment">#             )</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     connection.Connection = PatchedConnection</span></span><br><span class="line"><span class="comment">#     # also imported as a  global in pyppeteer.launcher</span></span><br><span class="line"><span class="comment">#     launcher.Connection = PatchedConnection</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># class DownloaderMiddleware(object):</span></span><br><span class="line"><span class="comment">#     # Not all methods need to be defined. If a method is not defined,</span></span><br><span class="line"><span class="comment">#     # scrapy acts as if the downloader middleware does not modify the</span></span><br><span class="line"><span class="comment">#     # passed objects.</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def __init__(self):</span></span><br><span class="line"><span class="comment">#         # print(&quot;Init downloaderMiddleware use pypputeer.&quot;)</span></span><br><span class="line"><span class="comment">#         # os.environ[&#x27;PYPPETEER_CHROMIUM_REVISION&#x27;] = &#x27;588429&#x27;</span></span><br><span class="line"><span class="comment">#         # pyppeteer.DEBUG = False</span></span><br><span class="line"><span class="comment">#         print(os.environ.get(&#x27;PYPPETEER_CHROMIUM_REVISION&#x27;))</span></span><br><span class="line"><span class="comment">#         loop = asyncio.get_event_loop()</span></span><br><span class="line"><span class="comment">#         task = asyncio.ensure_future(self.getbrowser())</span></span><br><span class="line"><span class="comment">#         loop.run_until_complete(task)</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#         # self.browser = task.result()</span></span><br><span class="line"><span class="comment">#         # print(self.browser)</span></span><br><span class="line"><span class="comment">#         # print(self.page)</span></span><br><span class="line"><span class="comment">#         # self.page = await browser.newPage()</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     async def getbrowser(self):</span></span><br><span class="line"><span class="comment">#         ua=redis_ua.sget(&quot;user-agent&quot;)</span></span><br><span class="line"><span class="comment">#         redis_ua.rpush(&#x27;user-agent&#x27;,ua)</span></span><br><span class="line"><span class="comment">#         # proxies=redis_ua.sget(&quot;daxiangIP&quot;)</span></span><br><span class="line"><span class="comment">#         # redisconn.lpush(&#x27;proxies:ipx&#x27;,proxies)</span></span><br><span class="line"><span class="comment">#         # Proxies=str(proxies,encoding=&#x27;utf-8&#x27;)</span></span><br><span class="line"><span class="comment">#         # print(ua,Proxies,&quot;&gt;&quot;*30)</span></span><br><span class="line"><span class="comment">#         # self.browser = await pyppeteer.launch(&#123;&#x27;headless&#x27;: False,&#x27;timeout&#x27;:0, </span></span><br><span class="line"><span class="comment">#         #                                   &#x27;args&#x27;: [</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--window-size=&#123;1300&#125;,&#123;600&#125;&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--disable-extensions&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--hide-scrollbars&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--disable-bundled-ppapi-flash&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--mute-audio&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--no-sandbox&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--disable-setuid-sandbox&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--disable-gpu&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--disable-infobars&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                       &#x27;--proxy-server=http://http-dyn.abuyun.com:9020&#x27;,</span></span><br><span class="line"><span class="comment">#         #                                   ],</span></span><br><span class="line"><span class="comment">#         #                                   &#x27;dumpio&#x27;: True</span></span><br><span class="line"><span class="comment">#         #                                   &#125;)</span></span><br><span class="line"><span class="comment">#         self.browser = await pyppeteer.connect(&#123;&#x27;browserWSEndpoint&#x27;: &#x27;ws://172.20.3.221:3000?--proxy-server=http://http-dyn.abuyun.com:9020&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #     &#x27;headless&#x27;: False,&#x27;timeout&#x27;:0,</span></span><br><span class="line"><span class="comment">#         # #                                 #     &#x27;args&#x27;: [</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--window-size=&#123;1300&#125;,&#123;600&#125;&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--disable-extensions&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--hide-scrollbars&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--disable-bundled-ppapi-flash&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--mute-audio&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--no-sandbox&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--disable-setuid-sandbox&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--disable-gpu&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--disable-infobars&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #       &#x27;--proxy-server=http://http-dyn.abuyun.com:9020&#x27;,</span></span><br><span class="line"><span class="comment">#         # #                                 #   ],</span></span><br><span class="line"><span class="comment">#         # #                                 #   &#x27;dumpio&#x27;: True</span></span><br><span class="line"><span class="comment">#                                             &#125;)</span></span><br><span class="line"><span class="comment">#         self.page = await self.browser.newPage()</span></span><br><span class="line"><span class="comment">#         # await page.setExtraHTTPHeaders(&#123;&#x27;Proxy-Authorization&#x27;: &#x27;Basic H6587BH09900664D:20C4314AF6C62E0F&#x27;&#125;)</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#         await self.page.authenticate(&#123;&quot;username&quot;: &quot;H6587BH09900664D&quot;, &quot;password&quot;: &quot;20C4314AF6C62E0F&quot;&#125;)          </span></span><br><span class="line"><span class="comment">#         # await self.page.setExtraHTTPHeaders(&quot;http://http-dyn.abuyun.com:9020&quot;) </span></span><br><span class="line"><span class="comment">#         # await self.page.setCookie(cookie)</span></span><br><span class="line"><span class="comment">#         await self.page.setViewport(viewport=&#123;&#x27;width&#x27;: 1366, &#x27;height&#x27;: 768&#125;)</span></span><br><span class="line"><span class="comment">#         await self.page.setUserAgent(str(ua,encoding=&#x27;utf-8&#x27;))</span></span><br><span class="line"><span class="comment">#         # await self.page.setUserAgent(&quot;Chrome (AppleWebKit/537.1; Chrome50.0; Windows NT 6.3) AppleWebKit/537.36 (KHTML like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393&quot;)</span></span><br><span class="line"><span class="comment">#         await self.page.setJavaScriptEnabled(enabled=True)</span></span><br><span class="line"><span class="comment">#         await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#             &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperties(navigator,&#123; webdriver:&#123; get: () =&gt; false &#125; &#125;) &#125;&#x27;&#x27;&#x27;)  </span></span><br><span class="line"><span class="comment">#         await self.page.evaluate(&#x27;&#x27;&#x27;() =&gt;&#123; window.navigator.chrome = &#123; runtime: &#123;&#125;,  &#125;; &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#         await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#             &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperty(navigator, &#x27;languages&#x27;, &#123; get: () =&gt; [&#x27;en-US&#x27;, &#x27;en&#x27;] &#125;); &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#         await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#             &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperty(navigator, &#x27;plugins&#x27;, &#123; get: () =&gt; [1, 2, 3, 4, 5,6], &#125;); &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#         await self.page.waitFor(10000)</span></span><br><span class="line"><span class="comment">#         # cooki=redisconn.get(&quot;ali1688cookie&quot;)</span></span><br><span class="line"><span class="comment">#         # redisconn.rpush(&#x27;ali1688cookie&#x27;,cooki)</span></span><br><span class="line"><span class="comment">#         # cookieStr=str(cooki,encoding=&#x27;utf-8&#x27;)</span></span><br><span class="line"><span class="comment">#         # for cook in json.loads(cookieStr):</span></span><br><span class="line"><span class="comment">#         #     await self.page.setCookie(cook)</span></span><br><span class="line"><span class="comment">#         return self.page</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">#     @classmethod</span></span><br><span class="line"><span class="comment">#     def from_crawler(cls, crawler):</span></span><br><span class="line"><span class="comment">#         # This method is used by Scrapy to create your spiders.</span></span><br><span class="line"><span class="comment">#         s = cls()</span></span><br><span class="line"><span class="comment">#         crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)</span></span><br><span class="line"><span class="comment">#         return s</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def process_request(self, request, spider):</span></span><br><span class="line"><span class="comment">#         # Called for each request that goes through the downloader</span></span><br><span class="line"><span class="comment">#         # middleware.</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#         # Must either:</span></span><br><span class="line"><span class="comment">#         # - return None: continue processing this request</span></span><br><span class="line"><span class="comment">#         # - or return a Response object</span></span><br><span class="line"><span class="comment">#         # - or return a Request object</span></span><br><span class="line"><span class="comment">#         # - or raise IgnoreRequest: process_exception() methods of</span></span><br><span class="line"><span class="comment">#         #   installed downloader middleware will be called</span></span><br><span class="line"><span class="comment">#         loop = asyncio.get_event_loop()</span></span><br><span class="line"><span class="comment">#         task = asyncio.ensure_future(self.usePypuppeteer(request))</span></span><br><span class="line"><span class="comment">#         loop.run_until_complete(task)</span></span><br><span class="line"><span class="comment">#         # return task.result()</span></span><br><span class="line"><span class="comment">#         return HtmlResponse(url=request.url, body=task.result(), encoding=&quot;utf-8&quot;, request=request)</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     async def intercept_request(self,req):</span></span><br><span class="line"><span class="comment">#         if req.resourceType in [&#x27;image&#x27;, &#x27;media&#x27;, &#x27;eventsource&#x27;, &#x27;websocket&#x27;]:</span></span><br><span class="line"><span class="comment">#             await req.abort()</span></span><br><span class="line"><span class="comment">#         else:</span></span><br><span class="line"><span class="comment">#             await req.continue_()</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     async def intercept_response(self,res):</span></span><br><span class="line"><span class="comment">#         resourceType = res.request.resourceType</span></span><br><span class="line"><span class="comment">#         # if resourceType in [&#x27;xhr&#x27;, &#x27;fetch&#x27;]:</span></span><br><span class="line"><span class="comment">#         #     resp = await res.text()</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment">#     async def usePypuppeteer(self, request):</span></span><br><span class="line"><span class="comment">#         ws = self.browser.wsEndpoint</span></span><br><span class="line"><span class="comment">#         # print(ws,&quot;running...&quot;)</span></span><br><span class="line"><span class="comment">#         while True:</span></span><br><span class="line"><span class="comment">#             try:</span></span><br><span class="line"><span class="comment">#                 await self.page.goto(request.url,&#123;&#x27;timeout&#x27;:0,&#x27;waitUntil&#x27;: &#x27;domcontentloaded&#x27;&#125;)</span></span><br><span class="line"><span class="comment">#                 if R&#x27;/s?k=&#x27; in request.url :</span></span><br><span class="line"><span class="comment">#                     next_url = await self.page.querySelector(&#x27;.a-last&#x27;)</span></span><br><span class="line"><span class="comment">#                 else:</span></span><br><span class="line"><span class="comment">#                     next_url = await self.page.querySelector(&#x27;#title&#x27;)</span></span><br><span class="line"><span class="comment">#                 print(next_url)</span></span><br><span class="line"><span class="comment">#                 if not next_url:</span></span><br><span class="line"><span class="comment">#                     continue</span></span><br><span class="line"><span class="comment">#                 break</span></span><br><span class="line"><span class="comment">#             except Exception as ex:</span></span><br><span class="line"><span class="comment">#                 print(&quot;network error ...&gt;&gt;&gt;1&quot;)</span></span><br><span class="line"><span class="comment">#                 # await self.page.close()</span></span><br><span class="line"><span class="comment">#                 await self.usePypuppeteer(request)</span></span><br><span class="line"><span class="comment">#                 # self.page.close()</span></span><br><span class="line"><span class="comment">#                 # await self.browser.disconnect()  </span></span><br><span class="line"><span class="comment">#                 # self.browser = await pyppeteer.connect(&#123;&#x27;browserWSEndpoint&#x27;: ws&#125;)</span></span><br><span class="line"><span class="comment">#                 # ua=redis_ua.sget(&quot;user-agent&quot;)</span></span><br><span class="line"><span class="comment">#                 # redis_ua.rpush(&#x27;user-agent&#x27;,ua)</span></span><br><span class="line"><span class="comment">#                 # self.page = await self.browser.newPage()</span></span><br><span class="line"><span class="comment">#                 # await self.page.authenticate(&#123;&quot;username&quot;: &quot;H6587BH09900664D&quot;, &quot;password&quot;: &quot;20C4314AF6C62E0F&quot;&#125;)          </span></span><br><span class="line"><span class="comment">#                 # await self.page.setViewport(viewport=&#123;&#x27;width&#x27;: 1366, &#x27;height&#x27;: 768&#125;)</span></span><br><span class="line"><span class="comment">#                 # await self.page.setUserAgent(str(ua,encoding=&#x27;utf-8&#x27;))</span></span><br><span class="line"><span class="comment">#                 # await self.page.setJavaScriptEnabled(enabled=True)</span></span><br><span class="line"><span class="comment">#                 # await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#                 #     &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperties(navigator,&#123; webdriver:&#123; get: () =&gt; false &#125; &#125;) &#125;&#x27;&#x27;&#x27;)  </span></span><br><span class="line"><span class="comment">#                 # await self.page.evaluate(&#x27;&#x27;&#x27;() =&gt;&#123; window.navigator.chrome = &#123; runtime: &#123;&#125;,  &#125;; &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#                 # await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#                 #     &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperty(navigator, &#x27;languages&#x27;, &#123; get: () =&gt; [&#x27;en-US&#x27;, &#x27;en&#x27;] &#125;); &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#                 # await self.page.evaluate(</span></span><br><span class="line"><span class="comment">#                 #     &#x27;&#x27;&#x27;() =&gt;&#123; Object.defineProperty(navigator, &#x27;plugins&#x27;, &#123; get: () =&gt; [1, 2, 3, 4, 5,6], &#125;); &#125;&#x27;&#x27;&#x27;)</span></span><br><span class="line"><span class="comment">#                 # await self.page.waitFor(10000)</span></span><br><span class="line"><span class="comment">#                 # self.saveMongo(request,ex)</span></span><br><span class="line"><span class="comment">#                 continue</span></span><br><span class="line"><span class="comment">#         if R&#x27;/s?k=&#x27; in request.url :</span></span><br><span class="line"><span class="comment">#             await self.page.setRequestInterception(True)</span></span><br><span class="line"><span class="comment">#             self.page.on(&#x27;request&#x27;, self.intercept_request)</span></span><br><span class="line"><span class="comment">#             # self.page.on(&#x27;response&#x27;, self.intercept_response)</span></span><br><span class="line"><span class="comment">#             await self.page.evaluate(&#x27;window.scrollBy(0, document.body.scrollHeight)&#x27;)</span></span><br><span class="line"><span class="comment">#             await asyncio.sleep(3)</span></span><br><span class="line"><span class="comment">#         else:</span></span><br><span class="line"><span class="comment">#             # await asyncio.sleep(3)</span></span><br><span class="line"><span class="comment">#             #鼠标滚动到底</span></span><br><span class="line"><span class="comment">#             for i in range(0,6):</span></span><br><span class="line"><span class="comment">#                 await self.page.evaluate(&#x27;window.scrollBy(0, &#123;&#125;)&#x27;.format(800*i))</span></span><br><span class="line"><span class="comment">#                 await asyncio.sleep(4)</span></span><br><span class="line"><span class="comment">#             # await self.page.evaluate(&#x27;window.scrollBy(0, document.body.scrollHeight*1/2)&#x27;)</span></span><br><span class="line"><span class="comment">#             # await asyncio.sleep(3)</span></span><br><span class="line"><span class="comment">#         content = await self.page.content()</span></span><br><span class="line"><span class="comment">#         # content = await self.page.evaluate(&#x27;document.body.textContent&#x27;, force_expr=True) </span></span><br><span class="line"><span class="comment">#         return content</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def saveMongo(self, request, exception):</span></span><br><span class="line"><span class="comment">#         if exception:</span></span><br><span class="line"><span class="comment">#             print(request.url)</span></span><br><span class="line"><span class="comment">#             request_url=str(request.url)</span></span><br><span class="line"><span class="comment">#             if R&quot;www.amazon.com/s?k=&quot; in request_url:</span></span><br><span class="line"><span class="comment">#                 db = MongoDB(&quot;AddTask&quot;,&quot;Add_List&quot;)</span></span><br><span class="line"><span class="comment">#                 items=&#123;&#125;</span></span><br><span class="line"><span class="comment">#                 items[&quot;cid&quot;]=GetAmazonUrlCID(request_url)</span></span><br><span class="line"><span class="comment">#                 items[&#x27;kwURL&#x27;]=[]</span></span><br><span class="line"><span class="comment">#                 items[&#x27;Finished_time&#x27;]=parse(str(datetime.now())[0:19])</span></span><br><span class="line"><span class="comment">#                 items[&#x27;link_url&#x27;]=request_url</span></span><br><span class="line"><span class="comment">#                 items[&#x27;status&#x27;]=&quot;failed&quot;</span></span><br><span class="line"><span class="comment">#                 items[&#x27;error&#x27;]=str(exception)</span></span><br><span class="line"><span class="comment">#                 if db.getInfo(&#123;&#x27;id&#x27;:int(items[&quot;cid&quot;])&#125;):</span></span><br><span class="line"><span class="comment">#                     db.updateDict(&#123;&#x27;id&#x27;:int(items[&#x27;cid&#x27;])&#125;,&#123;&#x27;$set&#x27;:items&#125;)</span></span><br><span class="line"><span class="comment">#                 # print(&quot;*&quot;*100,str(exception))</span></span><br><span class="line"><span class="comment">#             else:</span></span><br><span class="line"><span class="comment">#                 db_ = MongoDB(&quot;AmazonDB&quot;,&quot;AmazonContent&quot;)</span></span><br><span class="line"><span class="comment">#                 items=&#123;&#125;</span></span><br><span class="line"><span class="comment">#                 items[&#x27;id&#x27;]=db_.getID(&quot;amazonUrl&quot;)</span></span><br><span class="line"><span class="comment">#                 items[&quot;cid&quot;]=GetAmazonUrlCID(request_url)</span></span><br><span class="line"><span class="comment">#                 items[&#x27;Finished_time&#x27;]=parse(str(datetime.now())[0:19])</span></span><br><span class="line"><span class="comment">#                 items[&#x27;link_url&#x27;]=request_url</span></span><br><span class="line"><span class="comment">#                 items[&#x27;status&#x27;]=&quot;failed&quot;</span></span><br><span class="line"><span class="comment">#                 items[&#x27;error&#x27;]=str(exception)</span></span><br><span class="line"><span class="comment">#                 db_.insertDict(items)</span></span><br><span class="line"><span class="comment">#             return None</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def process_response(self, request, response, spider):</span></span><br><span class="line"><span class="comment">#         # Called with the response returned from the downloader.</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#         # Must either;</span></span><br><span class="line"><span class="comment">#         # - return a Response object</span></span><br><span class="line"><span class="comment">#         # - return a Request object</span></span><br><span class="line"><span class="comment">#         # - or raise IgnoreRequest</span></span><br><span class="line"><span class="comment">#         return response</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def process_exception(self, request, exception, spider):</span></span><br><span class="line"><span class="comment">#         # Called when a download handler or a process_request()</span></span><br><span class="line"><span class="comment">#         # (from other downloader middleware) raises an exception.</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#         # Must either:</span></span><br><span class="line"><span class="comment">#         # - return None: continue processing this exception</span></span><br><span class="line"><span class="comment">#         # - return a Response object: stops process_exception() chain</span></span><br><span class="line"><span class="comment">#         # - return a Request object: stops process_exception() chain</span></span><br><span class="line"><span class="comment">#         pass</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#     def spider_opened(self, spider):</span></span><br><span class="line"><span class="comment">#         spider.logger.info(&#x27;Spider opened: %s&#x27; % spider.name)</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># _patch_pyppeteer()</span></span><br></pre></td></tr></table></figure>











      
    </div>

    
    
    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  

      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/29/HyperLedger-Fabric-2-0-release%E6%B5%8B%E8%AF%95%E7%BD%91%E7%BB%9C%E9%83%A8%E7%BD%B2/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          
            <a href="/2020/07/29/HyperLedger-Fabric-2-0-release%E6%B5%8B%E8%AF%95%E7%BD%91%E7%BB%9C%E9%83%A8%E7%BD%B2/" class="post-title-link" itemprop="url">HyperLedger Fabric 2.0-release测试网络部署</a>
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-29 15:24:06" itemprop="dateCreated datePublished" datetime="2020-07-29T15:24:06+00:00">2020-07-29</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/block-chain/" itemprop="url" rel="index"><span itemprop="name">block chain</span></a>
                </span>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/29/HyperLedger-Fabric-2-0-release%E6%B5%8B%E8%AF%95%E7%BD%91%E7%BB%9C%E9%83%A8%E7%BD%B2/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/29/HyperLedger-Fabric-2-0-release%E6%B5%8B%E8%AF%95%E7%BD%91%E7%BB%9C%E9%83%A8%E7%BD%B2/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>934</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>1 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
          <h2 id="1-环境准备"><a href="#1-环境准备" class="headerlink" title="1.环境准备"></a>1.环境准备</h2><p>CentOS 7<br>Docker 18.09.4<br>Docker-compose 1.32.2<br>GO 1.13.4</p>
<h2 id="2-下载源码"><a href="#2-下载源码" class="headerlink" title="2.下载源码"></a>2.下载源码</h2><h3 id="1-创建go工作目录"><a href="#1-创建go工作目录" class="headerlink" title="1.创建go工作目录"></a>1.创建go工作目录</h3><p>mkdir go<br>mkdir go/src<br>mkdir go/pkg<br>mkdir go/bin<br>export GOPATH=xx/go<br>复制代码2.创建hyperledger目录<br>mkdir go/src/github.com/hyperledger<br>复制代码3.下载fabric release-2.0源码<br>cd go/src/github.com/hyperledger<br>git clone <a target="_blank" rel="noopener" href="https://github.com/hyperledger/fabric.git">https://github.com/hyperledger/fabric.git</a><br>cd fabric &amp;&amp; git checkout release-2.0<br>复制代码假如觉得git clone慢可参考 如何快速clone github代码库</p>
<h2 id="3-编译二进制文件以及docker镜像"><a href="#3-编译二进制文件以及docker镜像" class="headerlink" title="3. 编译二进制文件以及docker镜像"></a>3. 编译二进制文件以及docker镜像</h2><p>当前在fabric目录<br>打开控制台进入fabric目录，执行以下命令<br>make all<br>复制代码中间可能提示没有安装gcc，此时只需要安装即可<br>yum install gcc<br>复制代码执行完成后，查看编译二进制文件如下：<br>ll build/bin<br>复制代码控制台输出如下：</p>
<p>执行完成后，查看编译Docker镜像如下：<br>docker images |grep 2.0|grep fabric<br>复制代码控制台输出如下：</p>
<h2 id="4-运行测试网络"><a href="#4-运行测试网络" class="headerlink" title="4.运行测试网络"></a>4.运行测试网络</h2><p>1.将编译完成的二进制文件复制到 fabric-samples 目录<br>cp -r build/bin fabric-samples<br>复制代码2.网络准备<br>cd  fabric-samples/first-network<br>./byfn.sh generate<br>复制代码生成证书文件以及通道文件如下</p>
<p>3.运行测试网络<br>./byfn.sh up<br>复制代码控制台输出如下提示即运行成功</p>
<p>查看docker状态如下：<br>docker ps |grep fabric<br>复制代码执行结果如下：</p>
<h2 id="5-END"><a href="#5-END" class="headerlink" title="5.END"></a>5.END</h2><p>官方测试网络运行结束，接下来将对2.0的部署配置、合约以及raft共识进行继续学习，请持续关注。<br>推荐阅读： <a target="_blank" rel="noopener" href="https://blog.csdn.net/qq_28540443/article/details/104282768">Fabric2.0 first-network</a> 生成配置说明</p>
<p>链接：<a target="_blank" rel="noopener" href="https://juejin.im/post/5e440073518825490d124c92">https://juejin.im/post/5e440073518825490d124c92</a></p>

      
    </div>

    
    
    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  

      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/29/Hyperledger-Fabric-%E6%90%AD%E5%BB%BA%E7%AC%AC%E4%B8%80%E5%80%8B-Blockchain%E2%80%9D/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          
            <a href="/2020/07/29/Hyperledger-Fabric-%E6%90%AD%E5%BB%BA%E7%AC%AC%E4%B8%80%E5%80%8B-Blockchain%E2%80%9D/" class="post-title-link" itemprop="url">Hyperledger Fabric 搭建第一個 Blockchain” </a>
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-29 14:19:10" itemprop="dateCreated datePublished" datetime="2020-07-29T14:19:10+00:00">2020-07-29</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/block-chain/" itemprop="url" rel="index"><span itemprop="name">block chain</span></a>
                </span>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/29/Hyperledger-Fabric-%E6%90%AD%E5%BB%BA%E7%AC%AC%E4%B8%80%E5%80%8B-Blockchain%E2%80%9D/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/29/Hyperledger-Fabric-%E6%90%AD%E5%BB%BA%E7%AC%AC%E4%B8%80%E5%80%8B-Blockchain%E2%80%9D/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>16k</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>14 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
          <p>前言<br>只需单击即可启动功能完善的区块链网络<br>新的 IBM Blockchain Starter Membership Plan 为您提供了一种经济的方式来启动区块链网络。然后，您可以轻松部署区块链应用，比如这里描述的投票应用。</p>
<p>区块链（Blockchain）技术正在迅速发展，各行各业都正以极大的热情拥抱它。作者相信区块链将成为对信息科技领域产生革命性影响的一项新技术。目前，Hyperledger Fabric 正是此领域一个重要的技术框架与平台。</p>
<p>本文既是 Hyperledger Fabric（以下简称 Fabric）的实用教程，也是其学习、研究笔记。读者可以与作者一起，一步一步地学习 Fabric 基础知识，利用 Hyperledger Composer（以下简称 Composer）搭建 Fabric 本地开发环境，运行示例应用；并进一步分析、深入了解其技术结构与特点。</p>
<p>本文尽量不去简单复述其他相关文档，着重从实用角度与读者一起迅速建立对 Fabric 的直观认识，并在实践中逐渐理解区块链技术。本系列文章有三个部分，此为第一部分。</p>
<p>在本文示例中，Fabric 的版本为 1.1.0，Composer 的版本为 0.19.1。在以后的学习过程，它们可能随时有版本更新，而 micro 版本的改动应该对示例操作不会有影响。但仍请注意版本更新可能带来的变化。</p>
<p>基本概念与知识<br>基本概念与重要平台、技术、工具<br>用一句话来描述区块链：区块链是一个共享的不可修改的账本，可用来记录一个网络上所有交易的历史。这里所说的”交易”翻译自 “Transaction”。也可以将之称为”事务”，本文将之统一称为”交易”。</p>
<p>Hyperledger 是 Linux 基金会主持的一个开源项目，启动于 2015 年，其核心目标是建立开放的、标准化的、企业级的、能支持商业交易的分布式账本的框架与基础代码。</p>
<p>Hyperledger Fabric 是 Hyperledger 项目的一个组成部分，是一个区块链框架的实现。它将成为区块链应用开发、解决方案的基础。Fabric 框架支持组件化、可插拔的共识服务（Consensus Service）、成员服务（Membership Service）；”许可（Permissioned）”特性使之为”私有性”、”保密性”提供了可靠的解决方案；智能合约（Smart Contracts）在 Fabric 中通过”Chaincode”得以实现。Fabric 最初是由 Digital Asset、IBM 贡献给 Hyperledger 项目的。</p>
<p>Hyperledger Composer 也是 Hyperledger 项目的一个组成部分。通过它，人们可以更快速、容易地建立区块链业务模型，进行区块链网络及应用的开发、部署，并与现有系统、数据进行集成。</p>
<p>本文中，对于 Fabric 的学习，正是以 Composer 作为入口与基础工具，这样学习的效率更高。在学习初期，Fabric 与 Composer 的知识是紧密结合的，在后期，我们会对这两项技术分别深入学习、研究。</p>
<p>知识准备<br>区块链技术涉及到的技术比较多，本文希望能帮助读者将焦点一直放在区块链本身上，即使对某项技术学习得不是非常深入，也不会影响对于 Fabric 的学习。一般来说，希望读者还是能对以下知识预先有所了解：Ubuntu，Docker，Node.js，Javascript，npm，CA。</p>
<p>安装 Fabric 之前的环境准备<br>Ubuntu 16.04 LTS 64-bit<br>Fabric 支持 MacOSX、*nix 或者 Windows 10 操作系统；Composer 支持 Ubuntu Linux 14.04 / 16.04 LTS (64-bit)或者 MacOS 10.12 操作系统。</p>
<p>现在，我们使用 Ubuntu 16.04 LTS 64-bit ，作为我们的区块链部署系统。对于 Ubuntu 系统的安装与管理，这里不再详述，但可以通过以下命令确认版本信息：</p>
<p>确认 Ubuntu Linux 版本</p>
<h1 id="cat-etc-issue"><a href="#cat-etc-issue" class="headerlink" title="cat /etc/issue"></a>cat /etc/issue</h1><p>Ubuntu 16.04.4 LTS \n \l</p>
<h1 id="uname-a"><a href="#uname-a" class="headerlink" title="uname -a"></a>uname -a</h1><p>显示更多<br>系统用户<br>我们在学习过程中，几乎所有操作都通过一个专门的用户来完成（在本文示例中，使用的用户名为：fabric；读者可以根据需要使用自己的用户名，但请注意在后续示例中要相应修改）。请不要使用 root 用户。</p>
<p>添加一个新用户</p>
<h1 id="adduser-fabric"><a href="#adduser-fabric" class="headerlink" title="adduser fabric"></a>adduser fabric</h1><p>显示更多<br>将此用户加入 sudo 用户组</p>
<h1 id="usermod-aG-sudo-fabric"><a href="#usermod-aG-sudo-fabric" class="headerlink" title="usermod -aG sudo fabric"></a>usermod -aG sudo fabric</h1><p>显示更多<br>切换为 fabric 用户，并进入用户目录</p>
<h1 id="su-fabric"><a href="#su-fabric" class="headerlink" title="su fabric"></a>su fabric</h1><p>$ cd ~</p>
<p>显示更多<br>安装 Node.js, npm, Docker, Docker Compose, Python<br>现在需要安装 Node.js, Docker 等软件，Hyperledger 提供了一个脚本，可以用来自动安装。</p>
<p>下载并执行自动安装脚本<br>$ curl -O <a target="_blank" rel="noopener" href="https://hyperledger.github.io/composer/latest/prereqs-ubuntu.sh">https://hyperledger.github.io/composer/latest/prereqs-ubuntu.sh</a><br>$ chmod u+x prereqs-ubuntu.sh<br>$ ./prereqs-ubuntu.sh</p>
<p>显示较少<br>安装成功后，会显示以下内容，包括安装的软件名称及版本号。（后续版本可能会有变化。）</p>
<p>Installation completed, versions installed are:<br>Node:             v8.11.1<br>npm:              5.8.0<br>Docker:           Docker version 18.03.0-ce, build 0520e24<br>Docker Compose:   docker-compose version 1.13.0, build 1719ceb<br>Python:           Python 2.7.12<br>Please logout then login before continuing.</p>
<p>显示更多<br>退出并重新登录<br>请 退出 当前用户会话， 关闭客户端工具与 Ubuntu 的连接； 并 重新 以用户 fabric 登录 U b untu ， 以 使系统设置生效。</p>
<p>如果是自行手动安装这几项软件但版本号并不完全一致，可能会给后续过程带来一些障碍。所以，为节约时间，请尽量使用这个自动安装脚本；或手动安装这些版本的软件。</p>
<p>使用 Hyperledger Composer 安装 Fabric Runtime<br>现在，终于要正式开始安装 Composer 和 Fabric 了。</p>
<p>安装 Composer<br>Hyperledger Composer 是一个开放的开发框架、工具集，可以帮助人们更容易地开发、部署区块链应用。它支持 Fabric，并提供 Javascript SDK。我们可以通过 npm 来安装它的一系列组件。</p>
<p>安装 Composer 命令行工具 CLI<br>$ npm install -g composer-cli<br>$ npm view composer-cli version<br>0.19.1</p>
<p>显示更多<br>当前，其版本为：0.19.1。</p>
<p>我们之后许多操作（安装、部署、管理）都将通过 Composer CLI 完成。</p>
<p>因为之前的 Node.js 是通过 nvm 安装的（请参考：prereqs-ubuntu.sh），并在这里使用了”-g”选项，所以默认设置下安装的 node modules 文件可以在这里找到：</p>
<p>~/.nvm/versions/node/v8.11.1/lib/node_modules</p>
<p>安装 Composer REST Server<br>$ npm install -g composer-rest-server<br>$ npm view composer-rest-server version<br>0.19.1</p>
<p>显示更多<br>Composer REST server 可以根据我们开发、部署的区块链应用自动生成一些 RESTful API 接口，以方便通过浏览器、curl 等工具对之进行访问。</p>
<p>安装 generator-hyperledger-composer<br>$ npm install -g generator-hyperledger-composer<br>$ npm view generator-hyperledger-composer version<br>0.19.1</p>
<p>显示更多<br>它包含了一组 Yeoman generator，可以在 Yeoman 中执行，以根据模板生成我们将要部署的区块链网络文件。</p>
<p>安装 Yeoman<br>$ npm install -g yo<br>$ npm view yo version<br>2.0.2</p>
<p>显示更多<br>Yeoman 能根据定义好的 generator 迅速生成我们所需要的项目、应用的框架。</p>
<p>安装 Hyperledger Fabric Runtime<br>新建一个 Fabric Tools 目录<br>$ cd ~<br>$ mkdir fabric-tools<br>$ cd fabric-tools/</p>
<p>显示更多<br>fabric-tools 目录是我们以后的工作目录，读者可以按需要改成自己期望的目录名。</p>
<p>下载 Fabric Dev Server<br>$ curl -O <a target="_blank" rel="noopener" href="https://raw.githubusercontent.com/hyperledger/composer-">https://raw.githubusercontent.com/hyperledger/composer-</a><br>tools/master/packages/fabric-dev-servers/fabric-dev-servers.tar.gz</p>
<p>显示更多<br>目前，fabric-dev-servers.zip 包含了 Fabric1.0 与 Fabric1.1 的两套安装脚本，及用于初始化的 Fabric 相关配置。解压后文件位于 fabric-tools/fabric-scripts 目录下。</p>
<p>解压 Fabric Dev Server<br>$ tar -xvf fabric-dev-servers.tar.gz</p>
<p>显示更多<br>下载 Fabric Image 文件<br>$ ./downloadFabric.sh</p>
<p>显示更多<br>默认情况下这个脚本最终会执行 fabric-scripts/hlfv11/downloadFabric.sh，hlfv11 表示 Hyperledger Fabric V1.1。这个过程会下载 5 个 docker image 文件，共约 3.6G，视网络情况，可能需要比较长的时间。下载完成后可以通过 docker images 命令查看。</p>
<p>$ docker images<br>REPOSITORY                   TAG             IMAGE ID          CREATED             SIZE<br>hyperledger/fabric-ca        x86_64-1.1.0    72617b4fa9b4      2 weeks ago         299MB<br>hyperledger/fabric-orderer   x86_64-1.1.0    ce0c810df36a      2 weeks ago         180MB<br>hyperledger/fabric-peer      x86_64-1.1.0    b023f9be0771      2 weeks ago         187MB<br>hyperledger/fabric-ccenv     x86_64-1.1.0    c8b4909d8d46      2 weeks ago         1.39GB<br>hyperledger/fabric-couchdb   x86_64-0.4.6    7e73c828fc5b      6 weeks ago         1.56GB</p>
<p>显示更多<br>到这里，我们就非常迅速、方便的完成了 Fabric 的下载，及部署环境的安装，得益于 Docker Container 技术，并不需要我们做复杂的配置。接下来将要开始部署一个示例应用。在之前，我们要先启动 Fabric，并生成 PeerAdmin card。</p>
<p>启动 Fabric<br>$ ./startFabric.sh</p>
<p>显示更多<br>startFabric.sh 最终会执行 ~/fabric-tools/fabric-scripts/hlfv11/startFabric.sh，里面有如下一行内容：</p>
<p>ARCH=$ARCH docker-compose -f “${DOCKER_FILE}” up -d</p>
<p>显示更多<br>打开~/fabric-tools/fabric-scripts/hlfv11/composer/docker-compose.yml 我们可以看到有如下四个 Docker 应用的配置：ca.org1.example.com(CA Node)，orderer.example.com(Orderer Node)，peer0.org1.example.com(Peer Node)，couchdb(Database)。它们启动成功后即意味着 Fabric 区块链网络的核心部分已经处于运行状态了。</p>
<p>在 startFabric.sh 中，还有以下内容：</p>
<p>docker exec peer0.org1.example.com peer channel create……<br>docker exec -e…… peer0.org1.example.com peer channel join……</p>
<p>显示更多<br>它们的作用是建立一个通道（Channel）并将刚启动的节点 peer0.org1.example.com 加入到这个通道。</p>
<p>通道（Channel）是 Fabric 中的重要概念与设计，它是网络成员间通讯的私有的子网络；网络中会有多个通道同时存在；每个交易都在认证、授权后在某个通道里执行；所有数据、交易、成员、通道信息都只对此通道的授权成员可见。</p>
<p>生成并导入 PeerAdmin Card<br>$ ./createPeerAdminCard.sh</p>
<p>显示更多<br>这个脚本会生成一个 Card 文件，它包含了 Fabric 网络的信息以及管理员 PeerAdmin 与之连接所必须的信息；即管理员的身份证明文件；生成后这个文件会被导入到 Composer，你可以在~/.composer/cards/PeerAdmin@hlfv1 目录下找到被导入的 PeerAdmin Card 的文件内容。之后，Composer 会利用这个 Card 文件建立起到 Fabric 网络的连接。</p>
<p>在以后的学习中，我们会介绍如何建立一个自定义的 Card 文件。</p>
<p>部署第一个 Fabric 区块链业务网络<br>Fabric runtime 已经被成功安装、启动了，现在我们要部署第一个 Fabric 区块链业务网络。</p>
<p>准备业务网络<br>为使学习过程更容易，我们直接利用 Yeoman 及已经下载的 Generator 生成区块链网络框架。</p>
<p>在以后的章节中，我们会介绍如何按步骤手工完成定义、部署过程。</p>
<p>生成业务网络定义（Business Network Definition – BND）<br>$ yo hyperledger-composer:businessnetwork</p>
<p>显示更多<br>参数 businessnetwork 来自于之前安装的 generator-hyperledger-composer，表示了一组对应的模板文件。可以在 ~/.nvm/versions/node/v8.11.1/lib/node_modules/generator-hyperledger-composer/generators/businessnetwork/templates/ 下找到即将生成的内容的模板。</p>
<p>运行会提示输入相关信息，为方便学习，建议 Business network name 定义为：tutorial-network；Namespace 定义为：org.example.biznet，其他信息可以自行决定内容。将定义的内容如下：</p>
<p>Business network name: tutorial-network<br>Description: The first blockchain network<br>Author name: Alice<br>Author email: <a href="mailto:&#97;&#108;&#x69;&#x63;&#x65;&#x40;&#x6f;&#x72;&#x67;&#46;&#x65;&#120;&#x61;&#109;&#112;&#108;&#101;&#46;&#98;&#x69;&#x7a;&#110;&#101;&#x74;">&#97;&#108;&#x69;&#x63;&#x65;&#x40;&#x6f;&#x72;&#x67;&#46;&#x65;&#120;&#x61;&#109;&#112;&#108;&#101;&#46;&#98;&#x69;&#x7a;&#110;&#101;&#x74;</a><br>License: Apache-2.0<br>Namespace: org.example.biznet</p>
<p>显示更多<br>执行成功后，在当前 ~/fabric-tools/ 目录下，新增了一个目录 tutorial-network, 里面有 index.js, package.json 等文件，以及 lib, models 两个目录，这就是将要部署的区块链业务网络定义。</p>
<p>进入 tutorial-network 目录<br>$ cd ~/fabric-tools/tutorial-network</p>
<p>显示更多<br>后续的操作基本都在此目录下完成。</p>
<p>生成 .bna 文件<br>$ composer archive create -t dir -n .</p>
<p>显示更多<br>执行成功后，在当前目录下会产生一个新文件 <a href="mailto:&#116;&#117;&#116;&#x6f;&#x72;&#x69;&#97;&#108;&#x2d;&#x6e;&#x65;&#x74;&#119;&#x6f;&#x72;&#x6b;&#x40;&#48;&#46;&#48;&#46;&#49;&#x2e;&#x62;&#x6e;&#97;">&#116;&#117;&#116;&#x6f;&#x72;&#x69;&#97;&#108;&#x2d;&#x6e;&#x65;&#x74;&#119;&#x6f;&#x72;&#x6b;&#x40;&#48;&#46;&#48;&#46;&#49;&#x2e;&#x62;&#x6e;&#97;</a>，即是 tutorial-network 目录下文件的压缩包。解压后发现其中主要有文件及目录：lib/，models/，package.json，permissions.acl。我们会在以后详细解释、操作这些文件。</p>
<p>部署及启动业务网络<br>这是第一个示例区块链网络安装部署的最后一部分内容了。</p>
<p>部署示例业务网络 tutorial-network<br>$ composer network install –card PeerAdmin@hlfv1 –archiveFile <a href="mailto:&#116;&#x75;&#x74;&#x6f;&#114;&#105;&#97;&#x6c;&#x2d;&#110;&#101;&#116;&#x77;&#111;&#x72;&#107;&#x40;&#x30;&#46;&#48;&#46;&#x31;&#x2e;&#98;&#110;&#x61;">&#116;&#x75;&#x74;&#x6f;&#114;&#105;&#97;&#x6c;&#x2d;&#110;&#101;&#116;&#x77;&#111;&#x72;&#107;&#x40;&#x30;&#46;&#48;&#46;&#x31;&#x2e;&#98;&#110;&#x61;</a></p>
<p>显示更多<br>或者</p>
<p>$ composer network install –c PeerAdmin@hlfv1 –a <a href="mailto:&#x74;&#117;&#116;&#111;&#114;&#105;&#x61;&#108;&#45;&#x6e;&#101;&#116;&#x77;&#111;&#114;&#x6b;&#x40;&#x30;&#46;&#x30;&#46;&#49;&#46;&#x62;&#110;&#97;">&#x74;&#117;&#116;&#111;&#114;&#105;&#x61;&#108;&#45;&#x6e;&#101;&#116;&#x77;&#111;&#114;&#x6b;&#x40;&#x30;&#46;&#x30;&#46;&#49;&#46;&#x62;&#110;&#97;</a></p>
<p>显示更多<br>“composer network install” 命令会部署指定的 .bna 文件到 Fabric 网络。.bna 文件包括了这个业务网络的 Assets 模型、交易事务逻辑、访问控制规则等定义，但它并不能直接在 Fabric 上运行。.bna 文件是由 Composer 生成的，它是用 Composer 提供支持的一系列建模语言、规范定义的业务网络定义，我们必须将它先安装在 Fabric Peer 节点上。然后才可以在这个节点上启动运行这个业务网络。</p>
<p>参数 -c (–card) 应指定为在上一步骤中生成 PeerAdmin Card 文件。</p>
<p>参数 -a (–archiveFile) 应指定为将要部署的业务网络文件包。</p>
<p>启动业务网络<br>$ composer network start –networkName tutorial-network –networkVersion 0.0.1 –<br>networkAdmin admin –networkAdminEnrollSecret adminpw –card PeerAdmin@hlfv1 –file<br>networkadmin.card</p>
<p>显示更多<br>“composer network start” 用指定的 Card 启动这个网络；同时会生成一个当前业务网络的管理员 Card 文件，即此示例中的 networkadmin.card。</p>
<p>这个文件是 zip 格式的压缩文件，解压缩后，可以发现包含两个文件：connection.json, metadata.json。其中，metadata.json 内容如下：</p>
<p>{“version”:1,<br>“userName”:”admin”,<br>“businessNetwork”:”tutorial-network”,<br>“enrollmentSecret”:”adminpw”}</p>
<p>显示更多<br>“admin”, “tutorial-network” 正是我们此前的定义的管理员用户名，及业务网络名。我们在以后可以通过类似 -c admin@tutorial-network 使用这个管理员身份。</p>
<p>导入 tutorial-network 管理员 Card<br>$ composer card import –file networkadmin.card</p>
<p>显示更多<br>此时再查看 ~/.composer/cards 会发现新导入的 Card 文件。也可以直接通过命令查看。</p>
<p>查看导入的 Card 文件<br>$ composer card list<br>┌────────────────────────┬───────────┬──────────────────┐<br>│ Card Name              │ UserId    │ Business Network │<br>├────────────────────────┼───────────┼──────────────────┤<br>│ admin@tutorial-network │ admin     │ tutorial-network │<br>├────────────────────────┼───────────┼──────────────────┤<br>│ PeerAdmin@hlfv1        │ PeerAdmin │                  │<br>└────────────────────────┴───────────┴──────────────────┘</p>
<p>显示更多<br>确认 tutorial-network 安装成功<br>$ composer network ping –card admin@tutorial-network</p>
<p>显示更多<br>如果部署成功，会显示类似如下内容：</p>
<p>The connection to the network was successfully tested: tutorial-network<br>Business network version: 0.0.1<br>Composer runtime version: 0.19.1<br>participant: org.hyperledger.composer.system.NetworkAdmin#admin<br>identity: org.hyperledger.composer.system.Identity#…<br>Command succeeded</p>
<p>显示更多<br>启动 REST Server<br>$ composer-rest-server</p>
<p>显示更多<br>并在之后提示的选项中输入内容如下：</p>
<p>Card Name: admin@tutorial-network<br>Never use namespace<br>Enable authentication (default): No<br>Enable event publication over WebSockets (default): Yes<br>Enable TSL (default): No</p>
<p>显示更多<br>也可以直接带参数运行命令行：</p>
<p>$ composer-rest-server -c admin@tutorial-network -n never</p>
<p>显示更多<br>composer-rest-server 会根据部署的业务网络生成一系列 REST API,以方便用户通过浏览器或其他类似 curl 的应用程序访问这个区块链业务网络。如果在本机，我们可以通过这样的地址访问：</p>
<p><a target="_blank" rel="noopener" href="http://localhost:3000/explorer%E3%80%82">http://localhost:3000/explorer。</a></p>
<p>部署成功<br>到这里，我们就成功地通过 Hyperledger Composer 安装了 Hyperledger Fabric，并部署、启动了第一个区块链业务网络 tutorial-network。</p>
<p>访问区块链网络<br>现在我们可以开始访问部署成功的第一个 Fabric 区块链业务网络。</p>
<p>本文主要介绍通过浏览器和 curl 命令访问 REST Service。</p>
<p>通过浏览器访问 REST Service<br>在浏览器中输入 <a target="_blank" rel="noopener" href="http://fabric11dev1:3000/explorer%E3%80%82">http://fabric11dev1:3000/explorer。</a></p>
<p>fabric11dev1 是当前部署运行 Composer REST Server 的机器名。如果从本机访问，可以使用 localhost， 或者 直接使用 IP 地址访问。后文不再对此特别说明。</p>
<p>图 1. 入口界面<br>入口界面</p>
<p>SampleAsset 是这个区块链业务网络中定义的资产模型 (Asset)。表示一种有形或无形的可改变、转移的商品。</p>
<p>SampleParticipant 是这个业务网络的成员 (Participant)，可以拥有 Asset，提交 Transaction。</p>
<p>SampleTransaction 是一种交易或事务 (Transaction)，由成员提交到业务网络，用以改变、转移商品，或触发其他操作。</p>
<p>这些内容都可点击展开，显示 REST API 中对于此项内容的所有操作。</p>
<p>图 2. Participant 操作<br>Participant 操作</p>
<p>点击 GET ， POST 等按钮，可以进行展开并进行各种操作。</p>
<p>点击 POST 按钮，可添加一个新的 SampleParticipant。</p>
<p>图 3. 添加 SampleParticipant<br>添加 SampleParticipant</p>
<p>在 data 输入框中，输入如下内容：</p>
<p>{<br>“$class”: “org.example.biznet.SampleParticipant”,<br>“participantId”: “SP_1”,<br>“firstName”: “Alice”,<br>“lastName”: “Fabric”<br>}</p>
<p>显示更多<br>“org.example.biznet.SampleParticipant” 是将要添加 Participant 的模型名称；”SP_1″ 是当前业务网络中这种实例的唯一 ID。</p>
<p>我们会在以后详细分析这个模型的定义方法。</p>
<p>点击 Try it out! 按钮，REST Service 会向这个区块链业务网络添加新的 SampleParticipant。如果添加成功，会在界面下方显示 “Response Code 200” 类似的输出内容。</p>
<p>添加成功后，再点击第一个操作 GET ，展开后点击 Try it out! 按钮，即可列出所有 SampleParticipant。</p>
<p>图 4. 列出所有 SampleParticipant<br>列出所有 SampleParticipant</p>
<p>图 5. Asset 操作<br>Asset 操作</p>
<p>点击 GET ， POST 等按钮，可以进行 SampleAsset 各种操作。</p>
<p>点击 POST 按钮，可添加一个新的 SampleAsset。</p>
<p>图 6. 添加一个新的 SampleAsset<br>添加一个新的 SampleAsset</p>
<p>在 data 输入框中，输入如下内容：</p>
<p>{<br>“$class”: “org.example.biznet.SampleAsset”,<br>“assetId”: “SA_1”,<br>“owner”: “org.example.biznet.SampleParticipant#SP_1”,<br>“value”: “$100”<br>}</p>
<p>显示更多<br>“org.example.biznet.SampleAsset” 是将要添加实例的模型名；”SA_1″ 是当前业务网络中这种实例的唯一 ID。”org.example.biznet.SampleParticipant#SP_1″ 是对于之前建立的 ID 为 SP_1 的 SampleParticipant 的引用。</p>
<p>我们会在以后详细分析这个模型的定义方法。</p>
<p>点击 Try it out! 按钮，REST Service 会向这个区块链业务网络添加新的 SampleAsset。如果添加成功，会在界面下方显示 “Response Code 200” 类似的输出内容。</p>
<p>按此方法，我们可以再添加另一个 SampleAsset，其 ID 为 SA_2。</p>
<p>添加成功后，再点击第一个操作 GET ，展开后点击 Try it out! 按钮，即可列出所有 SampleAsset。</p>
<p>图 7. 列出所有 SampleAsset<br>列出所有 SampleAsset</p>
<p>在 Response Body 中，可以发现以 JSON 格式列出的所有 SampleAsset 实例。</p>
<p>图 8. 交易（Transaction）操作<br>交易（Transaction）操作</p>
<p>点击 GET ， POST 等按钮，可以进行交易的各种操作。</p>
<p>点击 POST 按钮，可提交一个新的交易。</p>
<p>图 9. 提交交易（Transaction）<br>提交交易（Transaction）</p>
<p>在 data 输入框中输入以下内容：</p>
<p>{<br>“$class”: “org.example.biznet.SampleTransaction”,<br>“asset”: “org.example.biznet.SampleAsset#SA_1”,<br>“newValue”: “$105”<br>}</p>
<p>显示更多<br>“org.example.biznet.SampleTransaction” 是将要提交的交易模型名称；”org.example.biznet. SampleTransaction#SA_1″ 是之前添加的 SampleAsset 实例的引用；newValue”$105″ 是指将这个 SampleAsset 实例中的 value 改成新的值 “$105″。</p>
<p>点击 Try it out! 按钮，REST Service 会向这个区块链业务网络提交这个交易。如果提交成功，会在界面下方显示 “Response Code 200” 类似的输出内容。</p>
<p>提交成功后，我们再次操作 SampleAsset – GET， 即可发现 SampleAsset#SA_1 的 value 已经被改为 “$105″。</p>
<p>“newValue” 这个操作的具体逻辑是在 ~/fabric-tools/tutorial-network/lib/logic.js 中定义的。其中所使用的 API，如 getAssetRegistry、getFactory 正是 Composer 提供的 Javascript API，这段 JS 会由被先前安装部署在 Peer 上的 Composer 业务网络执行。</p>
<p>在 SampleTransaction – GET 中，点击 Try it out! ，即可列出所有提交成功的交易。</p>
<p>图 10. 列出所有交易<br>列出所有交易</p>
<p>这里，每个交易比之前提交的内容，多了 transactionId, timestamp 两个内容，这是 tutorial-network 在处理交易时自动添加的。</p>
<p>图 11. System 操作<br>System 操作</p>
<p>通过 System – GET /system/historian 操作，可以列出这个业务网络的历史操作记录。部分内容如下：</p>
<p>[<br>{<br>“$class”: “org.hyperledger.composer.system.HistorianRecord”,<br>“transactionId”: “0d5eaeb63fe045505b1ff6d45083f6f7703a8ad7e6f2dd3b86190fab212ca0aa”,<br>“transactionType”: “org.hyperledger.composer.system.AddAsset”,<br>“transactionInvoked”:<br>“resource:org.hyperledger.composer.system.AddAsset#0d5eaeb63fe045505b1ff6d45083f6f7703a8ad7e6f2dd3b86190fab212ca0aa”,<br>“participantInvoking”: “resource:org.hyperledger.composer.system.NetworkAdmin#admin”,<br>“identityUsed”:<br>“resource:org.hyperledger.composer.system.Identity#0b0c50ab485d4b849a3050fcf630211ea4c9edb26c9e2b3308a1df4dc3dc77d6”,<br>“eventsEmitted”: [],<br>“transactionTimestamp”: “2018-03-31T10:09:24.709Z”<br>},<br>{<br>“$class”: “org.hyperledger.composer.system.HistorianRecord”,<br>“transactionId”: “6dcb8eb6-34aa-4b6e-95f6-08dc0b7285fb”,<br>“transactionType”: “org.hyperledger.composer.system.StartBusinessNetwork”,<br>“transactionInvoked”:<br>“resource:org.hyperledger.composer.system.StartBusinessNetwork#6dcb8eb6-34aa-4b6e-95f6-08dc0b7285fb”,<br>“eventsEmitted”: [],<br>“transactionTimestamp”: “2018-03-31T06:04:06.500Z”<br>},<br>{<br>“$class”: “org.hyperledger.composer.system.HistorianRecord”,<br>“transactionId”: “57b5004a3f865c84a2d298ec2d37bf3d1113ed61efad62055d11f1da0e3d2c1f”,<br>“transactionType”: “org.example.biznet.SampleTransaction”,<br>“transactionInvoked”:<br>“resource:org.example.biznet.SampleTransaction#57b5004a3f865c84a2d298ec2d37bf3d1113ed61efad62055d11f1da0e3d2c1f”,<br>“participantInvoking”: “resource:org.hyperledger.composer.system.NetworkAdmin#admin”,<br>“identityUsed”:<br>“resource:org.hyperledger.composer.system.Identity#0b0c50ab485d4b849a3050fcf630211ea4c9edb26c9e2b3308a1df4dc3dc77d6”,<br>“eventsEmitted”: [<br>{<br>“$class”: “org.example.biznet.SampleEvent”,<br>“asset”: “resource:org.example.biznet.SampleAsset#SA_1”,<br>“oldValue”: “$100”,<br>“newValue”: “$105”,<br>“eventId”: “57b5004a3f865c84a2d298ec2d37bf3d1113ed61efad62055d11f1da0e3d2c1f#0”,<br>“timestamp”: “2018-03-31T10:14:31.353Z”<br>}<br>],<br>“transactionTimestamp”: “2018-03-31T10:14:31.353Z”<br>},<br>{<br>“$class”: “org.hyperledger.composer.system.HistorianRecord”,<br>“transactionId”: “cd87d7c3940a2bc96d640cbf53a35f6a4fbcf3be5c953667428084ef5b02dabf”,<br>“transactionType”: “org.hyperledger.composer.system.AddParticipant”,<br>“transactionInvoked”:<br>“resource:org.hyperledger.composer.system.AddParticipant#cd87d7c3940a2bc96d640cbf53a35f6a4fbcf3be5c953667428084ef5b02dabf”,<br>“participantInvoking”: “resource:org.hyperledger.composer.system.NetworkAdmin#admin”,<br>“identityUsed”:<br>“resource:org.hyperledger.composer.system.Identity#0b0c50ab485d4b849a3050fcf630211ea4c9edb26c9e2b3308a1df4dc3dc77d6”,<br>“eventsEmitted”: [],<br>“transactionTimestamp”: “2018-03-31T10:00:32.858Z”<br>}<br>]</p>
<p>显示更多<br>示例中有 3 个操作：AddAsset, AddParticipant, SampleTransaction, StartBusinessNetwork。</p>
<p>其中 SampleTransaction 是我们之前手工提交的 Transaction；而 AddAsset, AddParticipant, StartBusinessNetwork 则是业务网络自己产生的事务操作历史记录。由此我们可以知道，当前 tutorial-network 中的一些操作（如：添加 SampleAsset）其实是通过一系列 Composer 模型语言定义了一个操作，由 Composer 提交到 Fabric Peer Node，最终由业务网络执行完成的，即智能合约（Smart Contract）在 Fabric 中的实现：Chaincode。我们会在后续学习中深入理解这种机制。</p>
<p>按上述方法，我们们可以浏览这个简单的区块链网络所提供服务的所有 REST API。</p>
<p>使用 curl 访问 REST Service<br>以上介绍的在浏览器中对 REST Service 及区块链网络的访问，都可以通过 curl 及其他类似应用程序，通过控制台进行。</p>
<p>使用 curl 程序调用 REST API – GET SampleAsset<br>$ curl -X GET –header ‘Accept: application/json’<br>‘<a target="_blank" rel="noopener" href="http://fabric11dev1:3000/api/SampleAsset&#39;">http://fabric11dev1:3000/api/SampleAsset&#39;</a></p>
<p>显示更多<br>为方便起见，这里并没有指定任何用户信息，因为目前没有加入安全验证机制，我们会在以后的学习中专门讨论用户安全、权限控制问题。</p>
<p>执行成功后，会返回 JSON 格式的文本。</p>
<p>使用 curl 程序调用 REST API – POST SampleTransaction<br>$ curl -X POST –header ‘Content-Type: application/json’ –header ‘Accept:<br>application/json’ -d ‘{<br>“$class”: “org.example.biznet.SampleTransaction”,<br>“asset”: “org.example.biznet.SampleAsset#SA_1”,<br>“newValue”: “$110”<br>}’ ‘<a target="_blank" rel="noopener" href="http://fabric11dev1:3000/api/SampleTransaction&#39;">http://fabric11dev1:3000/api/SampleTransaction&#39;</a></p>
<p>显示更多<br>结语<br>我们一起学习了 Hyperledger Fabric，Composer 的基本概念，并使用 Composer 部署了一个 Fabric 开发环境，安装并运行了第一个区块链业务网络与应用。最后通过 REST Service 访问了这个区块链应用。</p>
<p>本文对于 Fabric 的的学习内容还是初步的，旨在帮助大家迅速建立对 Fabric 的直观映像，以便为下一步深入研究打好基础。</p>
<p>参考资源<br>参考 IBM Blockchain Dev Center ，查看 IBM 在区块链领域的最新信息。<br>参考 Hyperledger Projects ，了解开源项目 Hyperledger 的主要内容。<br>参考 Hyperledger Fabric Documentation ，了解开源项目 Fabric 的主要内容。<br>阅读相关 Code Pattern: 使用 Hyperledger Composer 处理事务事件<br><a target="_blank" rel="noopener" href="https://developer.ibm.com/zh/tutorials/cl-lo-hyperledger-fabric-study-notes1/">link</a></p>

      
    </div>

    
    
    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  

      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/29/Hyperledger-Fabric-1-4-%E5%BF%AB%E9%80%9F%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          
            <a href="/2020/07/29/Hyperledger-Fabric-1-4-%E5%BF%AB%E9%80%9F%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA/" class="post-title-link" itemprop="url"> Hyperledger Fabric 1.4 快速环境搭建</a>
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-29 13:57:08" itemprop="dateCreated datePublished" datetime="2020-07-29T13:57:08+00:00">2020-07-29</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/block-chain/" itemprop="url" rel="index"><span itemprop="name">block chain</span></a>
                </span>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/29/Hyperledger-Fabric-1-4-%E5%BF%AB%E9%80%9F%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/29/Hyperledger-Fabric-1-4-%E5%BF%AB%E9%80%9F%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>3.4k</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>3 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
          <p>自己的硕士研究方向和区块链有关，工程上一直以IBM的Hyperledger Fabric为基础进行开发，对该项目关注也有两年了。目前迎来了Hyperledger Fabric v1.4，这也是Fabric的第一个长期支持版本，因此也比较有代表性，故在此和大家分享一下自己的环境搭建过程。</p>
<p>附上v1.4的官方文档：<a target="_blank" rel="noopener" href="https://hyperledger-fabric.readthedocs.io/en/release-1.4/%E3%80%82">https://hyperledger-fabric.readthedocs.io/en/release-1.4/。</a></p>
<p>环境说明:</p>
<p>　　本人测试环境为腾讯云学生机（1Core/RAM 2G/ROM 50G），CentOS 7.5 64位。</p>
<h2 id="Golang-安装配置（非必须）"><a href="#Golang-安装配置（非必须）" class="headerlink" title="Golang 安装配置（非必须）"></a>Golang 安装配置（非必须）</h2><p>下载安装包</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line">mkdir download</span><br><span class="line">cd download</span><br><span class="line"><span class="comment"># 从国内站点下载合适的安装包</span></span><br><span class="line">wget https://studygolang.com/dl/golang/go1<span class="number">.12</span><span class="number">.4</span>.linux-amd64.tar.gz </span><br><span class="line">解压</span><br><span class="line"></span><br><span class="line">tar -C /usr/local/ -xzvf go1<span class="number">.12</span><span class="number">.4</span>.linux-amd64.tar.gz  <span class="comment">#解压到/usr/local/go目录下</span></span><br><span class="line">配置</span><br><span class="line"></span><br><span class="line">vim /etc/profile  <span class="comment"># 配置系统环境变量</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># 在最下方插入</span></span><br><span class="line"><span class="comment"># Golang 环境变量</span></span><br><span class="line">export GOROOT=/usr/local/go</span><br><span class="line">export GOPATH=/root/go</span><br><span class="line">export GOBIN=$GOPATH/<span class="built_in">bin</span></span><br><span class="line">export PATH=$PATH:$GOROOT/<span class="built_in">bin</span></span><br><span class="line">export PATH=$PATH:$GOPATH/<span class="built_in">bin</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#使配置的环境变量生效</span></span><br><span class="line">source /etc/profile </span><br><span class="line"><span class="comment">#检查是否配置正确</span></span><br><span class="line">go version</span><br></pre></td></tr></table></figure>
<p>说明：Go的环境并非运行的必须条件，但后期调试智能合约可能会用到，而且官网安装说明也提到了需要安装Go。</p>
<h2 id="Node-js-安装配置（非必须）"><a href="#Node-js-安装配置（非必须）" class="headerlink" title="Node.js 安装配置（非必须）"></a>Node.js 安装配置（非必须）</h2><p>通过nvm安装Node.js和npm，该步骤也不是运行的必须条件，但后期如果需要node-sdk来开发应用程序的话就需要node环境。（npm源切换可参考npm 淘宝源切换教程）</p>
<p>安装nvm（官方文档：<a target="_blank" rel="noopener" href="https://github.com/nvm-sh/nvm%EF%BC%89">https://github.com/nvm-sh/nvm）</a></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 安装后重启该会话或重新开一个会话即可生效</span></span><br><span class="line">curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0<span class="number">.34</span><span class="number">.0</span>/install.sh | bash</span><br><span class="line">通过nvm安装Node.js和npm</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="comment"># 查看当前支持的版本，fabricv1.4要求只支持8.x</span></span><br><span class="line">nvm ls-remote</span><br><span class="line"><span class="comment"># 安装当前8.x LTS的最新版本（会同时安装npm）</span></span><br><span class="line">nvm install <span class="number">8.16</span><span class="number">.0</span></span><br><span class="line"><span class="comment"># 检查node.js安装版本</span></span><br><span class="line">node -v</span><br><span class="line"><span class="comment"># 检查npm的安装版本</span></span><br><span class="line">npm -v</span><br></pre></td></tr></table></figure>
<h2 id="Docker-安装"><a href="#Docker-安装" class="headerlink" title="Docker 安装"></a>Docker 安装</h2><p>Docker安装社区版的就可以，参考官方文档：<a target="_blank" rel="noopener" href="https://docs.docker.com/install/linux/docker-ce/centos/">https://docs.docker.com/install/linux/docker-ce/centos/</a></p>
<p>参考官方文档的第一种通过仓库安装的方式（傻瓜操作的话第三种直接脚本安装更简单）。</p>
<p>p.s. 由于docker-hub镜像仓库在国外，可能会被墙或者很慢，建议大家参考daocloud加速器配置一下仓库，或者用云服务器的可以参考各家的docker仓库源（阿里Docker镜像库、腾讯Docker镜像库等）。</p>
<p>卸载旧版本</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line">sudo yum remove docker \</span><br><span class="line">                  docker-client \</span><br><span class="line">                  docker-client-latest \</span><br><span class="line">                  docker-common \</span><br><span class="line">                  docker-latest \</span><br><span class="line">                  docker-latest-logrotate \</span><br><span class="line">                  docker-logrotate \</span><br><span class="line">                  docker-engine</span><br><span class="line"></span><br><span class="line">安装docker-ce的必需条件</span><br><span class="line"></span><br><span class="line">sudo yum install -y yum-utils \</span><br><span class="line">  device-mapper-persistent-data \</span><br><span class="line">  lvm2</span><br><span class="line">设置稳定版的仓库</span><br><span class="line"></span><br><span class="line">sudo yum-config-manager \</span><br><span class="line">    --add-repo \</span><br><span class="line">    https://download.docker.com/linux/centos/docker-ce.repo</span><br><span class="line">安装docker-ce</span><br><span class="line"></span><br><span class="line">yum install docker-ce docker-ce-cli containerd.io</span><br><span class="line">启动docker</span><br><span class="line"></span><br><span class="line">sudo systemctl start docker</span><br><span class="line">测试docker是否安装成功</span><br><span class="line"></span><br><span class="line">sudo docker run hello-world</span><br></pre></td></tr></table></figure>
<h2 id="Docker-Compose-安装"><a href="#Docker-Compose-安装" class="headerlink" title="Docker Compose 安装"></a>Docker Compose 安装</h2><p>Fabric中节点容器编排使用的使docker-compose，故需要安装docker-compose。官方文档：<a target="_blank" rel="noopener" href="https://docs.docker.com/compose/install/">https://docs.docker.com/compose/install/</a></p>
<p>下载docker-compose</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">sudo curl -L <span class="string">&quot;https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)&quot;</span> -o /usr/local/<span class="built_in">bin</span>/docker-compose</span><br><span class="line">为docker-compose配置执行权限</span><br><span class="line"></span><br><span class="line">sudo chmod +x /usr/local/<span class="built_in">bin</span>/docker-compose</span><br><span class="line">检查是否安装成功</span><br><span class="line"></span><br><span class="line">docker-compose -v</span><br></pre></td></tr></table></figure>
<h2 id="源码获取"><a href="#源码获取" class="headerlink" title="源码获取"></a>源码获取</h2><p>通过go get 的方式获取源码，配合git切换分支</p>
<p>获取fabric源码</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 获取fabric源码</span></span><br><span class="line">go get -u github.com/hyperledger/fabric</span><br><span class="line"><span class="comment"># 进入目录 切换分支</span></span><br><span class="line">cd go/src/github.com/hyperledger/fabric</span><br><span class="line">git checkout v1<span class="number">.4</span><span class="number">.1</span></span><br><span class="line">获取fabric-sample源码</span><br><span class="line"></span><br><span class="line">go get github.com/hyperledger/fabric-samples</span><br><span class="line"><span class="comment"># 进入目录，切换分支</span></span><br><span class="line">cd go/src/github.com/hyperledger/fabric-samples/</span><br><span class="line">git checkout v1<span class="number">.4</span><span class="number">.1</span></span><br></pre></td></tr></table></figure>
<h2 id="获取镜像和二进制文件"><a href="#获取镜像和二进制文件" class="headerlink" title="获取镜像和二进制文件"></a>获取镜像和二进制文件</h2><p>我们通过fabric-sample测试</p>
<p>通过脚本获取docker镜像和可执行文件（建议事先配置好docker仓库）</p>
<p>cd go/src/github.com/hyperledger/fabric-samples/scripts/<br>#此处可能因为网络问题执行时间比较久<br>bash bootstrap.sh</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">## 运行测试</span><br><span class="line">通过fabric-sample中first-network示例测试，执行了一个A和B转账，中间多次查询余额的一个过程。</span><br><span class="line"></span><br><span class="line">docker运行需要权限，最好用root身份运行。</span><br><span class="line"></span><br><span class="line">找到first-network示例</span><br><span class="line">&#96;&#96;&#96;python</span><br><span class="line">cd go&#x2F;src&#x2F;github.com&#x2F;hyperledger&#x2F;fabric-samples&#x2F;first-network</span><br><span class="line">生成密钥文件</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[root@VM_0_39_centos first-network]# bash byfn.sh generate</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<h2 id="启动示例"><a href="#启动示例" class="headerlink" title="启动示例"></a>启动示例</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">[root@VM_0_39_centos first-network]<span class="comment"># bash byfn.sh up</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="number">90</span></span><br><span class="line">===================== Query successful on peer1.org2 on channel <span class="string">&#x27;mychannel&#x27;</span> ===================== </span><br><span class="line"></span><br><span class="line">========= All GOOD, BYFN execution completed =========== </span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> _____   _   _   ____   </span><br><span class="line">| ____| | \ | | |  _ \  </span><br><span class="line">|  _|   |  \| | | | | | </span><br><span class="line">| |___  | |\  | | |_| | </span><br><span class="line">|_____| |_| \_| |____/  </span><br></pre></td></tr></table></figure>
<h2 id="清除示例"><a href="#清除示例" class="headerlink" title="清除示例"></a>清除示例</h2><p>执行删除密钥文件，停止并删除容器等操作</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">[root@VM_0_39_centos first-network]<span class="comment"># bash byfn.sh down</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p><a target="_blank" rel="noopener" href="https://www.cnblogs.com/xiaoxt/p/10806720.html">link</a><br><a target="_blank" rel="noopener" href="https://juejin.im/post/5da738f851882505ca647153">link</a><br><a target="_blank" rel="noopener" href="https://developer.ibm.com/zh/tutorials/cl-lo-hyperledger-fabric-study-notes1/">IMB hyperledger-fabric</a></p>

      
    </div>

    
    
    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  

      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/29/%E5%A6%82%E4%BD%95%E7%A0%B4%E8%A7%A3%E5%B9%B6%E8%AE%BF%E9%97%AEHexo-Blog%E5%B7%B2%E5%8A%A0%E5%AF%86%E7%9A%84%E6%96%87%E7%AB%A0/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          
            <a href="/2020/07/29/%E5%A6%82%E4%BD%95%E7%A0%B4%E8%A7%A3%E5%B9%B6%E8%AE%BF%E9%97%AEHexo-Blog%E5%B7%B2%E5%8A%A0%E5%AF%86%E7%9A%84%E6%96%87%E7%AB%A0/" class="post-title-link" itemprop="url">如何破解并访问Hexo Blog已加密的文章</a>
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-29 10:42:32" itemprop="dateCreated datePublished" datetime="2020-07-29T10:42:32+00:00">2020-07-29</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Hack/" itemprop="url" rel="index"><span itemprop="name">Hack</span></a>
                </span>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/29/%E5%A6%82%E4%BD%95%E7%A0%B4%E8%A7%A3%E5%B9%B6%E8%AE%BF%E9%97%AEHexo-Blog%E5%B7%B2%E5%8A%A0%E5%AF%86%E7%9A%84%E6%96%87%E7%AB%A0/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/29/%E5%A6%82%E4%BD%95%E7%A0%B4%E8%A7%A3%E5%B9%B6%E8%AE%BF%E9%97%AEHexo-Blog%E5%B7%B2%E5%8A%A0%E5%AF%86%E7%9A%84%E6%96%87%E7%AB%A0/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>2.3k</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>2 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="How-to-crack-hexo-simple-encrypted-blog？"><a href="#How-to-crack-hexo-simple-encrypted-blog？" class="headerlink" title="How to crack hexo simple encrypted blog？"></a>How to crack hexo simple encrypted blog？</h1><h2 id="事发"><a href="#事发" class="headerlink" title="事发"></a>事发</h2><p>下午一起加班的小伙伴在折腾blog，然后问我的blog文章是怎么加密的，然后，就是惨案的开始（orz…）</p>
<p>我直接发了一个截图，并友情提示，这段代码大概不安全。然后成功激发小伙伴的兴趣，开始尝试越过密码。大概1-2分钟后，blog加密文章形同虚设。</p>
<h2 id="分析"><a href="#分析" class="headerlink" title="分析"></a>分析</h2><p>我的blog现在是用额hexo的the next主题加各种自定义特效，因为搭建时间其实很早，所以当时使用了一种简单的js逻辑来加密blog。下面我们来看下这段有安全问题的代码：<br>代码位置： yourblog/themes/next/layout/custom/head.swig</p>
<figure class="highlight"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">&lt;script&gt;</span><br><span class="line">    (function()&#123;</span><br><span class="line">        <span class="keyword">if</span>(<span class="string">&#x27;&#123;&#123; page.password &#125;&#125;&#x27;</span>)&#123;</span><br><span class="line">            <span class="keyword">if</span> (prompt(<span class="string">&#x27;请输入密码&#x27;</span>) !== <span class="string">&#x27;&#123;&#123; page.password &#125;&#125;&#x27;</span>)&#123;</span><br><span class="line">                alert(<span class="string">&#x27;密码错误&#x27;</span>);</span><br><span class="line">                history.back();</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;)();</span><br><span class="line">&lt;/script&gt;</span><br></pre></td></tr></table></figure>
<p>乍看是否觉得除了逻辑简单没什么问题，然而眼尖的小伙伴分析出了第一个隐患：history.back(); ，而我已知静态页面的安全隐患。接下来我们逐个分析。</p>
<h2 id="istory-back"><a href="#istory-back" class="headerlink" title="istory.back()"></a>istory.back()</h2><p>back() 方法可加载历史列表中的前一个 URL（如果存在）。如果你的前一个URL就是这个文章的URL，那么恭喜，第一种破解方法get。（不理解，看下一条）</p>
<h2 id="静态页面"><a href="#静态页面" class="headerlink" title="静态页面"></a>静态页面</h2><p>hexo g 会根据blog相关文件在public目录下生成blog的所有静态文件。既然是已经生成好的静态页面，实际上html中包含了那段js代码的。因为需要获得html了之后才会触发那个页面访问加密的js，而理论上当我们获取到html页面之后，就可以停止当前页的js操作了。<br>That is to say, 你的blog word就是直接写在页面中的，虽然直接url去访问会弹alert，但是用curl直接获取这个页面的html，可以直接搜索到文章的访问密码, 分分钟攻破此种博文加密方式。上面利用history.back()的访问破解方式，归根结底也是这个问题引起的。</p>
<p>另外补充一点：因为用hexo的blog的人大多喜欢直接部署到github上利用github.io来展示，这样就要求使用Github的公开仓库，所以代码基本没有保密性可言。坊间流传直接看md文件我持疑，因为hexo d只会把public目录里生成的静态文件上传到github的仓库，除非是直接上传了blog项目的所有代码到github的仓库中。</p>
<h2 id="修复"><a href="#修复" class="headerlink" title="修复"></a>修复</h2><p>其实修复问题很好解决，既然简单的js加密不够安全，那我们就换一个安全的加密方式。<br>不知道怎么写？不慌，早已有大佬开发了插件hexo-blog-encrypt</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">安装hexo-blog-encrypt(安装方式<span class="number">2</span>选<span class="number">1</span>)</span><br><span class="line"><span class="number">1.</span>npm维护安装包</span><br><span class="line"></span><br><span class="line">npm install --save hexo-blog-encrypt (需要安装 npm)</span><br><span class="line"><span class="number">2.</span>yarn维护</span><br><span class="line"></span><br><span class="line">yarn add hexo-blog-encrypt (需要安装 Yarn)</span><br><span class="line">站点设置修改</span><br><span class="line">在 站点配置文件 _config.yml 中启用该插件</span><br></pre></td></tr></table></figure>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># encrypt plugin</span></span><br><span class="line">encrypt:</span><br><span class="line">  enable: true</span><br><span class="line">  default_abstract: 这是一篇加密文章，内容可能是个人日常吐槽或者特殊技术分享。如果你确实想看，请与我联系。非亲友团勿扰。</span><br><span class="line">  default_message: 输入密码，查看文章。</span><br></pre></td></tr></table></figure>
<h2 id="使用hexo-blog-encrypt加密blog"><a href="#使用hexo-blog-encrypt加密blog" class="headerlink" title="使用hexo-blog-encrypt加密blog"></a>使用hexo-blog-encrypt加密blog</h2><h3 id="使用-hexo-new-“yourblog”-后在页面添加password字段即可"><a href="#使用-hexo-new-“yourblog”-后在页面添加password字段即可" class="headerlink" title="使用 hexo new  “yourblog”  后在页面添加password字段即可"></a>使用 hexo new  “yourblog”  后在页面添加password字段即可</h3><p>在你的博文头部添加上相应字段，如 password, abstract, message。因为 abstract, message 在站点配置中设置了默认值，如果这里不想自定义 abstract, message，不用专门设置。<br>password: 代表blog的访问密码<br>abstract: 这篇博文的摘要，会显示在博客的列表页<br>message: 当查看加密博文时，密码输入框上面的提示性文字。建议使用中文，英文提示默认字体不好看，可以看我下面的截图。</p>
<p>visit_password</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">---</span><br><span class="line">title: 测试新文章加密</span><br><span class="line">date: </span><br><span class="line">category: hexo</span><br><span class="line">keywords: 博客文章密码</span><br><span class="line">password: yourpassword</span><br><span class="line">abstract: hexo密码测试</span><br><span class="line">message:  输入密码，查看文章</span><br><span class="line">---</span><br></pre></td></tr></table></figure>
<h2 id="反省总结"><a href="#反省总结" class="headerlink" title="反省总结"></a>反省总结</h2><p>这个事件，其实属于网络安全中最常见的问题“人的安全意识”，比如这个blog加密方案有访问漏洞，我是知道的，但是拖延症导致我没有足够重视这个问题（因为自己总是想着加密过的东西其实被看到也无妨，毕竟正要保密也不会发出来了balabala），结果一直没去修复（即使实际修复这个问题只需要10分钟），直到被小伙伴测试到bug才动手修复。</p>
<p>延伸到实际生产环境的安全问题，0 day漏洞被利用姑且还能辩解说是吃了信息不对等的亏。但也不乏一些有年头的机构的网站依然存在N day漏洞可利用，即使修复方案网上10s可能就能找到（例如：简单的打个补丁或者改几个配置项），这就真的是维护人员安全意识的问题了。所以在网络安全中，人的安全意识尤为重要，对于安全从业者更是如此，绝对不能有一丝松懈。<br><a target="_blank" rel="noopener" href="https://blog.fullstackpentest.com/how-to-crack-hexo-simple-encrypted-blog.html">link</a></p>

      
    </div>

    
    
    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  

      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/29/python-%E4%B8%89%E5%A4%A7%E7%A5%9E%E5%99%A8/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          
            <a href="/2020/07/29/python-%E4%B8%89%E5%A4%A7%E7%A5%9E%E5%99%A8/" class="post-title-link" itemprop="url">python 三大神器</a>
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-29 10:22:23" itemprop="dateCreated datePublished" datetime="2020-07-29T10:22:23+00:00">2020-07-29</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/python/" itemprop="url" rel="index"><span itemprop="name">python</span></a>
                </span>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/29/python-%E4%B8%89%E5%A4%A7%E7%A5%9E%E5%99%A8/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/29/python-%E4%B8%89%E5%A4%A7%E7%A5%9E%E5%99%A8/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>1.6k</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>1 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="Python-三大神器"><a href="#Python-三大神器" class="headerlink" title="Python 三大神器"></a>Python 三大神器</h1><h2 id="1-pip-用来包管理"><a href="#1-pip-用来包管理" class="headerlink" title="1. pip 用来包管理"></a>1. pip 用来包管理</h2><p>文档：<a target="_blank" rel="noopener" href="https://pip.pypa.io/en/latest/installing.html">https://pip.pypa.io/en/latest/installing.html</a></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">### 安装，可指定版本号</span></span><br><span class="line">(sudo) pip install Django==<span class="number">1.6</span><span class="number">.8</span></span><br><span class="line"> </span><br><span class="line"><span class="comment">### 升级</span></span><br><span class="line">(sudo) pip install bpython --upgrade</span><br><span class="line"> </span><br><span class="line"><span class="comment">### 一次安装多个</span></span><br><span class="line">(sudo) pip install BeautifulSoup4 fabric virtualenv</span><br><span class="line"> </span><br><span class="line">从文本中安装，文本中为包名，一行一个，可以指定版本号</span><br><span class="line">(sudo) pip install –r requirements.txt</span><br><span class="line"></span><br><span class="line"><span class="comment">### 删除</span></span><br><span class="line">(sudo) pip uninstall xlrd</span><br><span class="line"> </span><br><span class="line"><span class="comment">### 导出当前已经安装包</span></span><br><span class="line">pip freeze &gt; requirements.txt</span><br></pre></td></tr></table></figure>
<h2 id="2-virtualenv-独立Python环境管理"><a href="#2-virtualenv-独立Python环境管理" class="headerlink" title="2. virtualenv 独立Python环境管理"></a>2. virtualenv 独立Python环境管理</h2><p>文档： <a target="_blank" rel="noopener" href="http://virtualenvwrapper.readthedocs.org/en/latest/">http://virtualenvwrapper.readthedocs.org/en/latest/</a></p>
<p>virtualenv 是一个创建Python独立环境的包，virtualenvwrapper 使得virtualenv变得更好用</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">### 安装:</span></span><br><span class="line">(sudo) pip install virtualenv virtualenvwrapper</span><br><span class="line"> </span><br><span class="line"><span class="comment">### 修改.bash_profile 或 .zshrc（如果你用 zsh 的话），添加以下语句</span></span><br><span class="line">export WORKON_HOME=$HOME/.virtualenvs</span><br><span class="line">export PROJECT_HOME=$HOME/workspace</span><br><span class="line">source /usr/local/<span class="built_in">bin</span>/virtualenvwrapper.sh</span><br></pre></td></tr></table></figure>
<p>mkvirtualenv ENV：创建运行环境ENV</p>
<p>rmvirtualenv ENV：删除运行环境ENV</p>
<p>mkproject mic：创建mic项目和运行环境mic</p>
<p>mktmpenv：创建临时运行环境</p>
<p>workon bsp: 工作在bsp运行环境</p>
<p>lsvirtualenv: 列出可用的运行环境</p>
<p>lssitepackages: 列出当前环境安装了的包</p>
<p>创建的环境是独立的，互不干扰，无需sudo权限即可使用 pip 来进行包的管理。</p>
<h2 id="3-fabric-服务器管理和应用发布"><a href="#3-fabric-服务器管理和应用发布" class="headerlink" title="3. fabric 服务器管理和应用发布"></a>3. fabric 服务器管理和应用发布</h2><p>官网：<a target="_blank" rel="noopener" href="http://www.fabfile.org/">http://www.fabfile.org/</a></p>
<p><a target="_blank" rel="noopener" href="http://docs.fabfile.org/">文档：</a><br><a target="_blank" rel="noopener" href="https://fabric-chs.readthedocs.io/zh_CN/chs/tutorial.html">中文文档</a></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line">fabric: application deployment <span class="keyword">or</span> systems administration tasks</span><br><span class="line"></span><br><span class="line"><span class="comment">#coding:utf-8</span></span><br><span class="line"> </span><br><span class="line"><span class="keyword">from</span> fabric.api <span class="keyword">import</span> *</span><br><span class="line"> </span><br><span class="line"><span class="comment">### 服务器列表</span></span><br><span class="line">env.hosts = [<span class="string">&#x27;user@server1&#x27;</span>,<span class="string">&#x27;user2@server2&#x27;</span>]</span><br><span class="line"> </span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">ls_home</span>():</span></span><br><span class="line">    <span class="keyword">with</span> cd(<span class="string">&#x27;/home/bae/&#x27;</span>):</span><br><span class="line">        run(<span class="string">&#x27;ls&#x27;</span>)</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br><span class="line"><span class="string">常用命令</span></span><br><span class="line"><span class="string"> </span></span><br><span class="line"><span class="string">lcd(dir): 进入本机某目录</span></span><br><span class="line"><span class="string">local(cmd): 本机上执行命令</span></span><br><span class="line"><span class="string">cd(dir): 进入服务器某目录</span></span><br><span class="line"><span class="string">run(cmd):服务器上执行命令</span></span><br><span class="line"><span class="string">&#x27;&#x27;&#x27;</span></span><br></pre></td></tr></table></figure>
<p>把上面的文件保存成 fabfile.py 在终端上进入该文件的目录，执行</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">fab 函数名</span><br><span class="line">比如：</span><br><span class="line">fab ls_home</span><br></pre></td></tr></table></figure>
<h2 id="fabric-报错-fabric-Import-Error-cannot-import-name-‘isMappingType’"><a href="#fabric-报错-fabric-Import-Error-cannot-import-name-‘isMappingType’" class="headerlink" title="fabric  报错:  fabric Import Error: cannot import name ‘isMappingType’"></a>fabric  报错:  fabric Import Error: cannot import name ‘isMappingType’</h2><p><a target="_blank" rel="noopener" href="https://stackoverflow.com/questions/29306752/fabric-import-error-cannot-import-name-ismappingtype">解决原文</a><br>Until Python 3 implementation of fabric is released, you can also use any of the available forks.</p>
<p>One of them is available in fabric3 pip package, which is compatible with Python 3:</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install fabric3 <span class="keyword">or</span> pip3 install fabric3</span><br></pre></td></tr></table></figure>

<p>更多使用方法请参见官方文档。</p>

      
    </div>

    
    
    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  

      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="http://blog.hory-ai.com/2020/07/28/pyppeteer-%E5%9C%A8debugging%E7%8A%B6%E6%80%81%E4%B8%8B%E9%A9%B1%E5%8A%A8chrome%E6%B5%8F%E8%A7%88%E5%99%A8/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Hory Skone">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Horysk 宏睿时空">
    </span>
      <header class="post-header">
        <h2 class="post-title" itemprop="name headline">
          
            <a href="/2020/07/28/pyppeteer-%E5%9C%A8debugging%E7%8A%B6%E6%80%81%E4%B8%8B%E9%A9%B1%E5%8A%A8chrome%E6%B5%8F%E8%A7%88%E5%99%A8/" class="post-title-link" itemprop="url">pyppeteer 在debugging状态下驱动chrome浏览器</a>
        </h2>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2020-07-28 14:12:44" itemprop="dateCreated datePublished" datetime="2020-07-28T14:12:44+00:00">2020-07-28</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/pyppeteer/" itemprop="url" rel="index"><span itemprop="name">pyppeteer</span></a>
                </span>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2020/07/28/pyppeteer-%E5%9C%A8debugging%E7%8A%B6%E6%80%81%E4%B8%8B%E9%A9%B1%E5%8A%A8chrome%E6%B5%8F%E8%A7%88%E5%99%A8/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/07/28/pyppeteer-%E5%9C%A8debugging%E7%8A%B6%E6%80%81%E4%B8%8B%E9%A9%B1%E5%8A%A8chrome%E6%B5%8F%E8%A7%88%E5%99%A8/" itemprop="commentCount"></span>
    </a>
  </span>
  
  <br>
            <span class="post-meta-item" title="Symbols count in article">
              <span class="post-meta-item-icon">
                <i class="far fa-file-word"></i>
              </span>
                <span class="post-meta-item-text">Symbols count in article: </span>
              <span>2.6k</span>
            </span>
            <span class="post-meta-item" title="Reading time">
              <span class="post-meta-item-icon">
                <i class="far fa-clock"></i>
              </span>
                <span class="post-meta-item-text">Reading time &asymp;</span>
              <span>2 mins.</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
          <h2 id="1-以命令窗口启动chrome浏览器，选择远程连接的端口为9222"><a href="#1-以命令窗口启动chrome浏览器，选择远程连接的端口为9222" class="headerlink" title="1.以命令窗口启动chrome浏览器，选择远程连接的端口为9222"></a>1.以命令窗口启动chrome浏览器，选择远程连接的端口为9222</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">chrome.exe --disable-infobars --remote-debugging-port=<span class="number">9222</span> --user-data-<span class="built_in">dir</span>=<span class="string">&quot;设置路径&quot;</span></span><br><span class="line"> --disable-infobars 表示关闭提示</span><br></pre></td></tr></table></figure>
<h2 id="2-启动之后chromium通过http-localhost-9222-json得到调试信息，chrome通过http-127-0-0-1-9222-json-version-得到调试信息，"><a href="#2-启动之后chromium通过http-localhost-9222-json得到调试信息，chrome通过http-127-0-0-1-9222-json-version-得到调试信息，" class="headerlink" title="2. 启动之后chromium通过http://localhost:9222/json得到调试信息，chrome通过http://127.0.0.1:9222/json/version  得到调试信息，"></a>2. 启动之后chromium通过<a target="_blank" rel="noopener" href="http://localhost:9222/json%E5%BE%97%E5%88%B0%E8%B0%83%E8%AF%95%E4%BF%A1%E6%81%AF%EF%BC%8Cchrome%E9%80%9A%E8%BF%87http://127.0.0.1:9222/json/version">http://localhost:9222/json得到调试信息，chrome通过http://127.0.0.1:9222/json/version</a>  得到调试信息，</h2><p>其中webSocketDebuggerUrl为pyppeteer连接的ws地址。</p>
<p>页面显示为</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">&#123;</span><br><span class="line">   <span class="string">&quot;Browser&quot;</span>: <span class="string">&quot;Chrome/78.0.3904.70&quot;</span>,</span><br><span class="line">   <span class="string">&quot;Protocol-Version&quot;</span>: <span class="string">&quot;1.3&quot;</span>,</span><br><span class="line">   <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36&quot;</span>,</span><br><span class="line">   <span class="string">&quot;V8-Version&quot;</span>: <span class="string">&quot;7.8.279.17&quot;</span>,</span><br><span class="line">   <span class="string">&quot;WebKit-Version&quot;</span>: <span class="string">&quot;537.36 (@edb9c9f3de0247fd912a77b7f6cae7447f6d3ad5)&quot;</span>,</span><br><span class="line">   <span class="string">&quot;webSocketDebuggerUrl&quot;</span>: <span class="string">&quot;ws://127.0.0.1:9222/devtools/browser/8fc97fd6-a7dd-4ff2-b760-3f6b25b7419b&quot;</span></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h2 id="3-案例"><a href="#3-案例" class="headerlink" title="3.案例"></a>3.案例</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">from</span> pyppeteer.launcher <span class="keyword">import</span> connect</span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line">useragents=[<span class="string">&#x27;Mozilla/5.0 (Windows NT 6.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36&#x27;</span>,<span class="string">&#x27;Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36&#x27;</span>,<span class="string">&#x27;Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36&#x27;</span>,<span class="string">&#x27;Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36&#x27;</span>,<span class="string">&#x27;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36&#x27;</span>,<span class="string">&#x27;Mozilla/5.0 (Windows NT 6.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36&#x27;</span>,<span class="string">&#x27;Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36&#x27;</span>,<span class="string">&#x27;Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36&#x27;</span>]</span><br><span class="line"> </span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">main</span>(<span class="params">url</span>):</span></span><br><span class="line">    connect_params=&#123;</span><br><span class="line">        <span class="string">&#x27;browserWSEndpoint&#x27;</span>: <span class="string">&#x27;ws://127.0.0.1:9222/devtools/browser/8fc97fd6-a7dd-4ff2-b760-3f6b25b7419b&#x27;</span>,</span><br><span class="line"><span class="string">&#x27;logLevel&#x27;</span>:<span class="number">3</span>,</span><br><span class="line">    &#125;</span><br><span class="line">    browser = <span class="keyword">await</span> connect(connect_params)</span><br><span class="line">    page = <span class="keyword">await</span> browser.newPage()   <span class="comment"># 启动个新的浏览器页面</span></span><br><span class="line">    <span class="keyword">await</span> page.setUserAgent(random.choice(useragents))</span><br><span class="line">    <span class="comment">#设置页面超时时间</span></span><br><span class="line">    page.setDefaultNavigationTimeout(<span class="number">1000</span>*<span class="number">60</span>) <span class="comment">#60s</span></span><br><span class="line">    <span class="comment">#启用js</span></span><br><span class="line">    <span class="keyword">await</span> page.setJavaScriptEnabled(<span class="literal">True</span>)</span><br><span class="line">    <span class="keyword">await</span> page.setViewport(&#123;<span class="string">&#x27;width&#x27;</span>:<span class="number">1300</span>,<span class="string">&#x27;height&#x27;</span>:<span class="number">750</span>&#125;) <span class="comment">#设置界面</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        <span class="keyword">await</span> page.goto(url)    <span class="comment"># 访问登录页面</span></span><br><span class="line">    <span class="keyword">except</span> Exception <span class="keyword">as</span> e1:</span><br><span class="line">        print(<span class="string">&#x27;e1:&#x27;</span>,e1)</span><br><span class="line">        <span class="keyword">await</span> page.close()</span><br><span class="line">        <span class="keyword">await</span> browser.close()</span><br><span class="line">        <span class="keyword">return</span></span><br><span class="line">  </span><br><span class="line"> </span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    loop = asyncio.get_event_loop()</span><br><span class="line">    url = <span class="string">&#x27;https://www.baidu.com&#x27;</span></span><br><span class="line">    m = main(url)</span><br><span class="line">    loop.run_until_complete(m)</span><br></pre></td></tr></table></figure>
      
    </div>

    
    
    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  


  
  <nav class="pagination">
    <a class="extend prev" rel="prev" href="/"><i class="fa fa-angle-left" aria-label="Previous page"></i></a><a class="page-number" href="/">1</a><span class="page-number current">2</span><a class="page-number" href="/page/3/">3</a><span class="space">&hellip;</span><a class="page-number" href="/page/7/">7</a><a class="extend next" rel="next" href="/page/3/"><i class="fa fa-angle-right" aria-label="Next page"></i></a>
  </nav>



          </div>
          

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          Table of Contents
        </li>
        <li class="sidebar-nav-overview">
          Overview
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">Hory Skone</p>
  <div class="site-description" itemprop="description"></div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">67</span>
          <span class="site-state-item-name">posts</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">30</span>
        <span class="site-state-item-name">categories</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">35</span>
        <span class="site-state-item-name">tags</span></a>
      </div>
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/horysk" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;horysk" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
      </span>
      <span class="links-of-author-item">
        <a href="mailto:admin@horysk.com" title="E-Mail → mailto:admin@horysk.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>E-Mail</a>
      </span>
      <span class="links-of-author-item">
        <a href="http://www.hory-ai.com/" title="HoryAI → http:&#x2F;&#x2F;www.hory-ai.com" rel="noopener" target="_blank"><i class="fab fa-google fa-fw"></i>HoryAI</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://blog.csdn.net/sirobot" title="CSDN → https:&#x2F;&#x2F;blog.csdn.net&#x2F;sirobot" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>CSDN</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://www.zhihu.com/people/AI_HH" title="ZhiHu → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;AI_HH" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>ZhiHu</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://www.kaggle.com/" title="Kaggle → https:&#x2F;&#x2F;www.kaggle.com&#x2F;" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>Kaggle</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://tianchi.aliyun.com/competition/gameList/activeList" title="TianChi → https:&#x2F;&#x2F;tianchi.aliyun.com&#x2F;competition&#x2F;gameList&#x2F;activeList" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>TianChi</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://works.yangerxiao.com/honeyed-words-generator" title="土情话 → https:&#x2F;&#x2F;works.yangerxiao.com&#x2F;honeyed-words-generator" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>土情话</a>
      </span>
  </div>



      </div>

      
      <script type="text/javascript" charset="utf-8" src="/js/tagcloud.js"></script>
      <script type="text/javascript" charset="utf-8" src="/js/tagcanvas.js"></script>
      <div class="widget-wrap">
          <h3 class="widget-title">Tag Cloud</h3>
          <div id="myCanvasContainer" class="widget tagcloud">
              <canvas width="250" height="250" id="resCanvas" style="width:100%">
                  <ul class="tag-list" itemprop="keywords"><li class="tag-list-item"><a class="tag-list-link" href="/tags/AI/" rel="tag">AI</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/BI/" rel="tag">BI</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Centos/" rel="tag">Centos</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/DNS/" rel="tag">DNS</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Dapp/" rel="tag">Dapp</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Docker/" rel="tag">Docker</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Hack/" rel="tag">Hack</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Hexo/" rel="tag">Hexo</a><span class="tag-list-count">6</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/HyperLedger-Fabric/" rel="tag">HyperLedger Fabric</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Hyperledger-Fabric/" rel="tag">Hyperledger Fabric</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Iftop/" rel="tag">Iftop</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Linux/" rel="tag">Linux</a><span class="tag-list-count">9</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/ML/" rel="tag">ML</a><span class="tag-list-count">5</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/MTProxy/" rel="tag">MTProxy</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Mongo/" rel="tag">Mongo</a><span class="tag-list-count">4</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Network/" rel="tag">Network</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Pyppeteer/" rel="tag">Pyppeteer</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Tools/" rel="tag">Tools</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/VPN/" rel="tag">VPN</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/Vnc/" rel="tag">Vnc</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/baostock/" rel="tag">baostock</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/block-chain/" rel="tag">block chain</a><span class="tag-list-count">3</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/centos-xfce-vnc/" rel="tag">centos-xfce-vnc</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/docker/" rel="tag">docker</a><span class="tag-list-count">11</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/fabric/" rel="tag">fabric</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/hexo/" rel="tag">hexo</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/horysk/" rel="tag">horysk</a><span class="tag-list-count">3</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/k8s/" rel="tag">k8s</a><span class="tag-list-count">5</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/linux/" rel="tag">linux</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/mongo/" rel="tag">mongo</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/pyppeteer/" rel="tag">pyppeteer</a><span class="tag-list-count">2</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/python/" rel="tag">python</a><span class="tag-list-count">4</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/quant/" rel="tag">quant</a><span class="tag-list-count">7</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/stock/" rel="tag">stock</a><span class="tag-list-count">1</span></li><li class="tag-list-item"><a class="tag-list-link" href="/tags/tushare/" rel="tag">tushare</a><span class="tag-list-count">1</span></li></ul>
              </canvas>
          </div>
      </div>
      
    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        
  <div class="beian"><a href="http://www.beian.miit.gov.cn/" rel="noopener" target="_blank">null </a>
  </div>

<div class="copyright">
  
  &copy; 2015 – 
  <span itemprop="copyrightYear">2022</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Hory Skone</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-chart-area"></i>
    </span>
    <span title="Symbols count total">275k</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-coffee"></i>
    </span>
    <span title="Reading time total">4:10</span>
</div>

        
<div class="busuanzi-count">
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    <span class="post-meta-item" id="busuanzi_container_site_uv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-user"></i>
      </span>
      <span class="site-uv" title="Total Visitors">
        <span id="busuanzi_value_site_uv"></span>
      </span>
    </span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item" id="busuanzi_container_site_pv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-eye"></i>
      </span>
      <span class="site-pv" title="Total Views">
        <span id="busuanzi_value_site_pv"></span>
      </span>
    </span>
</div>








      </div>
    </footer>
  </div>

  
  
  <script color='255,255,255' opacity='0.6' zIndex='-1' count='99' src="/lib/canvas-nest/canvas-nest.min.js"></script>
  <script size="300" alpha="0.6" zIndex="-1" src="/lib/canvas-ribbon/canvas-ribbon.js"></script>
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>

<script src="/js/utils.js"></script>

<script src="/js/motion.js"></script>


<script src="/js/schemes/pisces.js"></script>


<script src="/js/next-boot.js"></script>

<script src="/js/bookmark.js"></script>


  <script defer src="/lib/three/three.min.js"></script>
    <script defer src="/lib/three/three-waves.min.js"></script>
    <script defer src="/lib/three/canvas_lines.min.js"></script>
    <script defer src="/lib/three/canvas_sphere.min.js"></script>


  
  <script>
    (function(){
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x=document.getElementsByTagName("link");
		//Find the last canonical URL
		if(x.length > 0){
			for (i=0;i<x.length;i++){
				if(x[i].rel.toLowerCase() == 'canonical' && x[i].href){
					canonicalURL=x[i].href;
				}
			}
		}
    //Get protocol
	    if (!canonicalURL){
	    	curProtocol = window.location.protocol.split(':')[0];
	    }
	    else{
	    	curProtocol = canonicalURL.split(':')[0];
	    }
      //Get current URL if the canonical URL does not exist
	    if (!canonicalURL) canonicalURL = window.location.href;
	    //Assign script content. Replace current URL with the canonical URL
      !function(){var e=/([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,r=canonicalURL,t=document.referrer;if(!e.test(r)){var n=(String(curProtocol).toLowerCase() === 'https')?"https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif":"//api.share.baidu.com/s.gif";t?(n+="?r="+encodeURIComponent(document.referrer),r&&(n+="&l="+r)):r&&(n+="?l="+r);var i=new Image;i.src=n}}(window);})();
  </script>




  
<script src="/js/local-search.js"></script>













  

  


<script>
NexT.utils.loadComments(document.querySelector('#valine-comments'), () => {
  NexT.utils.getScript('https://cdn.jsdelivr.net/npm/valine@1.3.9/dist/Valine.min.js', () => {
    var GUEST = ['nick', 'mail', 'link'];
    var guest = 'nick,mail,link';
    guest = guest.split(',').filter(item => {
      return GUEST.includes(item);
    });
    new Valine({
      el         : '#valine-comments',
      verify     : false,
      notify     : false,
      appId      : '2ASH47v4VIRijd91HeEi3pws-gzGzoHsz',
      appKey     : 'ojRfR89cSV614pgJFT36lFFB',
      placeholder: "昵称填写qq可以显示qq头像和昵称哦~",
      avatar     : 'monsterid',
      meta       : guest,
      pageSize   : '10' || 10,
      visitor    : false,
      lang       : '' || 'zh-cn',
      path       : location.pathname,
      recordIP   : true,
      serverURLs : '',
      requiredFields: ['nick','mail'] 
    });
  }, window.Valine);
});
</script>

  <!-- 页面点击特效 -->
  <script type="text/javascript" src="/js/jquery-3.3.1.min.js"></script>
  <script type="text/javascript" src="/js/click-word.js"></script>
  
  <script type="text/javascript"
  color="255,255,255" opacity='0.7' zIndex="-2" count="500" src="//cdn.bootcss.com/canvas-nest.js/1.0.0/canvas-nest.min.js"></script>
  
<script src="/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"pluginRootPath":"live2dw/","pluginJsPath":"lib/","pluginModelPath":"assets/","tagMode":false,"log":false,"model":{"jsonPath":"/live2dw/assets/z16.model.json"},"display":{"position":"right","width":300,"height":600},"mobile":{"show":true}});</script></body>
</html>
